You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by as...@apache.org on 2021/05/21 12:58:01 UTC
[airflow-site] branch master updated: Add documentation for Apache
Airflow 2.1.0 (#424)
This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow-site.git
The following commit(s) were added to refs/heads/master by this push:
new c6ff437 Add documentation for Apache Airflow 2.1.0 (#424)
c6ff437 is described below
commit c6ff4372047482ba30a19983f7ba985566d6d141
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Fri May 21 13:57:53 2021 +0100
Add documentation for Apache Airflow 2.1.0 (#424)
---
.../2.1.0}/.buildinfo | 2 +-
.../2.1.0/_api/airflow/exceptions/index.html} | 1073 +-
.../airflow/executors/base_executor/index.html} | 892 +-
.../airflow/executors/celery_executor/index.html} | 1476 +--
.../celery_kubernetes_executor/index.html} | 705 +-
.../airflow/executors/dask_executor/index.html} | 513 +-
.../airflow/executors/debug_executor/index.html} | 568 +-
.../executors/executor_constants/index.html} | 506 +-
.../airflow/executors/executor_loader/index.html} | 576 +-
.../2.1.0/_api/airflow/executors}/index.html | 355 +-
.../executors/kubernetes_executor/index.html} | 1473 +--
.../airflow/executors/local_executor/index.html} | 928 +-
.../executors/sequential_executor/index.html} | 491 +-
.../2.1.0/_api/airflow/hooks/S3_hook/index.html} | 432 +-
.../2.1.0/_api/airflow/hooks/base/index.html} | 687 +-
.../2.1.0/_api/airflow/hooks/base_hook/index.html} | 432 +-
.../2.1.0/_api/airflow/hooks/dbapi/index.html} | 961 +-
.../_api/airflow/hooks/dbapi_hook/index.html} | 432 +-
.../_api/airflow/hooks/docker_hook/index.html} | 432 +-
.../_api/airflow/hooks/druid_hook/index.html} | 432 +-
.../_api/airflow/hooks/filesystem/index.html} | 492 +-
.../2.1.0/_api/airflow/hooks/hdfs_hook/index.html} | 432 +-
.../_api/airflow/hooks/hive_hooks/index.html} | 432 +-
.../2.1.0/_api/airflow/hooks/http_hook/index.html} | 432 +-
.../2.1.0/_api/airflow/hooks/index.html} | 493 +-
.../2.1.0/_api/airflow/hooks/jdbc_hook/index.html} | 432 +-
.../_api/airflow/hooks/mssql_hook/index.html} | 432 +-
.../_api/airflow/hooks/mysql_hook/index.html} | 432 +-
.../_api/airflow/hooks/oracle_hook/index.html} | 432 +-
.../2.1.0/_api/airflow/hooks/pig_hook/index.html} | 432 +-
.../_api/airflow/hooks/postgres_hook/index.html} | 432 +-
.../_api/airflow/hooks/presto_hook/index.html} | 432 +-
.../_api/airflow/hooks/samba_hook/index.html} | 432 +-
.../_api/airflow/hooks/slack_hook/index.html} | 432 +-
.../_api/airflow/hooks/sqlite_hook/index.html} | 473 +-
.../_api/airflow/hooks/subprocess/index.html} | 550 +-
.../_api/airflow/hooks/webhdfs_hook/index.html} | 432 +-
.../_api/airflow/hooks/zendesk_hook/index.html} | 432 +-
.../2.1.0/_api/airflow/models/base/index.html} | 535 +-
.../_api/airflow/models/baseoperator/index.html | 2678 +++++
.../_api/airflow/models/connection/index.html} | 962 +-
.../2.1.0/_api/airflow/models/crypto/index.html} | 593 +-
.../2.1.0/_api/airflow/models/dag/index.html | 3454 ++++++
.../2.1.0/_api/airflow/models/dagbag/index.html} | 853 +-
.../2.1.0/_api/airflow/models/dagcode/index.html} | 758 +-
.../2.1.0/_api/airflow/models/dagparam/index.html} | 518 +-
.../_api/airflow/models/dagpickle/index.html} | 526 +-
.../2.1.0/_api/airflow/models/dagrun/index.html | 1777 +++
.../2.1.0/_api/airflow/models/errors/index.html} | 562 +-
.../2.1.0/_api/airflow/models/index.html | 10016 +++++++++++++++++
.../2.1.0/_api/airflow/models/log/index.html} | 631 +-
.../2.1.0/_api/airflow/models/pool/index.html} | 803 +-
.../airflow/models/renderedtifields/index.html} | 698 +-
.../_api/airflow/models/sensorinstance/index.html} | 790 +-
.../_api/airflow/models/serialized_dag/index.html} | 963 +-
.../_api/airflow/models/skipmixin/index.html} | 593 +-
.../2.1.0/_api/airflow/models/slamiss/index.html} | 633 +-
.../2.1.0/_api/airflow/models/taskfail/index.html} | 619 +-
.../_api/airflow/models/taskinstance/index.html | 2981 +++++
.../_api/airflow/models/taskmixin/index.html} | 629 +-
.../_api/airflow/models/taskreschedule/index.html} | 691 +-
.../2.1.0/_api/airflow/models/variable/index.html} | 817 +-
.../2.1.0/_api/airflow/models/xcom/index.html} | 798 +-
.../2.1.0/_api/airflow/models/xcom_arg/index.html} | 630 +-
.../2.1.0/_api/airflow/operators/bash/index.html} | 721 +-
.../airflow/operators/bash_operator/index.html} | 529 +-
.../_api/airflow/operators/branch/index.html} | 553 +-
.../airflow/operators/branch_operator/index.html} | 488 +-
.../airflow/operators/check_operator/index.html} | 562 +-
.../airflow/operators/dagrun_operator/index.html} | 529 +-
.../_api/airflow/operators/datetime/index.html} | 550 +-
.../airflow/operators/docker_operator/index.html} | 488 +-
.../operators/druid_check_operator/index.html} | 488 +-
.../2.1.0/_api/airflow/operators/dummy/index.html} | 552 +-
.../airflow/operators/dummy_operator/index.html} | 488 +-
.../2.1.0/_api/airflow/operators/email/index.html} | 633 +-
.../airflow/operators/email_operator/index.html} | 488 +-
.../_api/airflow/operators/gcs_to_s3/index.html} | 488 +-
.../airflow/operators/generic_transfer/index.html} | 594 +-
.../google_api_to_s3_transfer/index.html} | 513 +-
.../airflow/operators/hive_operator/index.html} | 488 +-
.../operators/hive_stats_operator/index.html} | 488 +-
.../airflow/operators/hive_to_druid/index.html} | 554 +-
.../airflow/operators/hive_to_mysql/index.html} | 513 +-
.../operators/hive_to_samba_operator/index.html} | 529 +-
.../airflow/operators/http_operator/index.html} | 488 +-
.../2.1.0/_api/airflow/operators/index.html} | 574 +-
.../airflow/operators/jdbc_operator/index.html} | 488 +-
.../_api/airflow/operators/latest_only/index.html} | 544 +-
.../operators/latest_only_operator/index.html} | 488 +-
.../airflow/operators/mssql_operator/index.html} | 488 +-
.../airflow/operators/mssql_to_hive/index.html} | 513 +-
.../airflow/operators/mysql_operator/index.html} | 488 +-
.../airflow/operators/mysql_to_hive/index.html} | 511 +-
.../airflow/operators/oracle_operator/index.html} | 488 +-
.../operators/papermill_operator/index.html} | 488 +-
.../airflow/operators/pig_operator/index.html} | 488 +-
.../operators/postgres_operator/index.html} | 488 +-
.../operators/presto_check_operator/index.html} | 545 +-
.../airflow/operators/presto_to_mysql/index.html} | 513 +-
.../2.1.0/_api/airflow/operators/python/index.html | 1542 +++
.../airflow/operators/python_operator/index.html} | 488 +-
.../operators/redshift_to_s3_operator/index.html} | 512 +-
.../s3_file_transform_operator/index.html} | 489 +-
.../operators/s3_to_hive_operator/index.html} | 511 +-
.../operators/s3_to_redshift_operator/index.html} | 513 +-
.../airflow/operators/slack_operator/index.html} | 488 +-
.../2.1.0/_api/airflow/operators/sql/index.html | 1680 +++
.../operators/sql_branch_operator/index.html} | 511 +-
.../airflow/operators/sqlite_operator/index.html} | 488 +-
.../_api/airflow/operators/subdag/index.html} | 739 +-
.../airflow/operators/subdag_operator/index.html} | 488 +-
.../airflow/operators/trigger_dagrun/index.html} | 638 +-
.../_api/airflow/operators/weekday/index.html} | 582 +-
.../_api/airflow/secrets/base_secrets/index.html} | 538 +-
.../secrets/environment_variables/index.html} | 475 +-
.../2.1.0/_api/airflow/secrets/index.html} | 590 +-
.../airflow/secrets/local_filesystem/index.html} | 710 +-
.../_api/airflow/secrets/metastore/index.html} | 618 +-
.../2.1.0/_api/airflow/sensors/base/index.html} | 914 +-
.../sensors/base_sensor_operator/index.html} | 438 +-
.../2.1.0/_api/airflow/sensors/bash/index.html} | 657 +-
.../_api/airflow/sensors/date_time/index.html} | 666 +-
.../airflow/sensors/date_time_sensor/index.html} | 438 +-
.../_api/airflow/sensors/external_task/index.html} | 731 +-
.../sensors/external_task_sensor/index.html} | 438 +-
.../_api/airflow/sensors/filesystem/index.html} | 667 +-
.../_api/airflow/sensors/hdfs_sensor/index.html} | 438 +-
.../sensors/hive_partition_sensor/index.html} | 438 +-
.../_api/airflow/sensors/http_sensor/index.html} | 438 +-
.../2.1.0/_api/airflow/sensors/index.html} | 499 +-
.../sensors/metastore_partition_sensor/index.html} | 480 +-
.../named_hive_partition_sensor/index.html} | 438 +-
.../2.1.0/_api/airflow/sensors/python/index.html} | 503 +-
.../_api/airflow/sensors/s3_key_sensor/index.html} | 438 +-
.../airflow/sensors/s3_prefix_sensor/index.html} | 438 +-
.../_api/airflow/sensors/smart_sensor/index.html} | 1612 +--
.../2.1.0/_api/airflow/sensors/sql/index.html} | 704 +-
.../_api/airflow/sensors/sql_sensor/index.html} | 438 +-
.../_api/airflow/sensors/time_delta/index.html} | 637 +-
.../airflow/sensors/time_delta_sensor/index.html} | 479 +-
.../_api/airflow/sensors/time_sensor/index.html} | 634 +-
.../airflow/sensors/web_hdfs_sensor/index.html} | 438 +-
.../2.1.0/_api/airflow/sensors/weekday/index.html} | 546 +-
docs-archive/apache-airflow/2.1.0/_api/index.html | 1 +
.../apache-airflow/2.1.0/_images/add-dag-tags.png | Bin 0 -> 68537 bytes
.../apache-airflow/2.1.0/_images/add-role.png | Bin 0 -> 30509 bytes
.../apache-airflow/2.1.0/_images/airflow.gif | Bin 0 -> 416302 bytes
.../apache-airflow/2.1.0/_images/apache.jpg | Bin 0 -> 43364 bytes
.../2.1.0/_images/arch-diag-basic.png | Bin 0 -> 16357 bytes
.../2.1.0/_images/arch-diag-kubernetes.png | Bin 0 -> 21804 bytes
.../2.1.0/_images/arch-diag-kubernetes2.png | Bin 0 -> 86384 bytes
.../2.1.0/_images/arch-diag-logging.png | Bin 0 -> 25843 bytes
.../apache-airflow/2.1.0/_images/basic-dag.png | Bin 0 -> 5393 bytes
.../apache-airflow/2.1.0/_images/branch_note.png | Bin 0 -> 31771 bytes
.../2.1.0/_images/branch_with_trigger.png | Bin 0 -> 34081 bytes
.../2.1.0/_images/branch_without_trigger.png | Bin 0 -> 39815 bytes
.../2.1.0/_images/cli_completion.gif | Bin 0 -> 775263 bytes
docs-archive/apache-airflow/2.1.0/_images/code.png | Bin 0 -> 240101 bytes
.../2.1.0/_images/connection_create.png | Bin 0 -> 52889 bytes
.../2.1.0/_images/connection_edit.png | Bin 0 -> 31186 bytes
.../apache-airflow/2.1.0/_images/connections.png | Bin 0 -> 79451 bytes
.../apache-airflow/2.1.0/_images/context.png | Bin 0 -> 124467 bytes
.../apache-airflow/2.1.0/_images/dag_doc.png | Bin 0 -> 40094 bytes
.../2.1.0/_images/dag_serialization.png | Bin 0 -> 99917 bytes
.../apache-airflow/2.1.0/_images/dags-page-new.png | Bin 0 -> 483599 bytes
.../apache-airflow/2.1.0/_images/dags-page-old.png | Bin 0 -> 493009 bytes
docs-archive/apache-airflow/2.1.0/_images/dags.png | Bin 0 -> 211122 bytes
.../default_instance_name_configuration.png | Bin 0 -> 154888 bytes
.../apache-airflow/2.1.0/_images/duration.png | Bin 0 -> 150876 bytes
.../2.1.0/_images/edge_label_example.png | Bin 0 -> 24592 bytes
.../example_instance_name_configuration.png | Bin 0 -> 154915 bytes
.../2.1.0/_images/example_passing_conf.png | Bin 0 -> 41080 bytes
.../apache-airflow/2.1.0/_images/gantt.png | Bin 0 -> 71033 bytes
.../2.1.0/_images/graph-view-new.png | Bin 0 -> 56973 bytes
.../2.1.0/_images/graph-view-old.png | Bin 0 -> 54884 bytes
.../apache-airflow/2.1.0/_images/graph.png | Bin 0 -> 110218 bytes
...iz-91fd3ca4f3dc01a69b3f84fbcd6b5c7975945ba4.png | Bin 0 -> 41708 bytes
...1fd3ca4f3dc01a69b3f84fbcd6b5c7975945ba4.png.map | 2 +
.../2.1.0/_images/k8s-failed-pod.png | Bin 0 -> 18937 bytes
.../2.1.0/_images/k8s-happy-path.png | Bin 0 -> 17257 bytes
.../2.1.0/_images/latest_only_with_trigger.png | Bin 0 -> 42887 bytes
.../apache-airflow/2.1.0/_images/new-role.png | Bin 0 -> 14115 bytes
.../2.1.0/_images/operator_extra_link.png | Bin 0 -> 90359 bytes
.../apache-airflow/2.1.0/_images/pin_large.png | Bin 0 -> 24922 bytes
.../2.1.0/_images/run_task_on_celery_executor.png | Bin 0 -> 55939 bytes
.../2.1.0/_images/smart_sensor_architecture.png | Bin 0 -> 80325 bytes
.../smart_sensor_single_task_execute_flow.png | Bin 0 -> 75462 bytes
.../apache-airflow/2.1.0/_images/subdag_after.png | Bin 0 -> 30245 bytes
.../apache-airflow/2.1.0/_images/subdag_before.png | Bin 0 -> 70382 bytes
.../apache-airflow/2.1.0/_images/subdag_zoom.png | Bin 0 -> 145017 bytes
.../apache-airflow/2.1.0/_images/task_doc.png | Bin 0 -> 245714 bytes
.../apache-airflow/2.1.0/_images/task_group.gif | Bin 0 -> 609981 bytes
.../2.1.0/_images/task_lifecycle_diagram.png | Bin 0 -> 135225 bytes
.../2.1.0/_images/task_manual_vs_scheduled.png | Bin 0 -> 5806 bytes
.../apache-airflow/2.1.0/_images/tree-view-new.png | Bin 0 -> 36934 bytes
.../apache-airflow/2.1.0/_images/tree-view-old.png | Bin 0 -> 21601 bytes
docs-archive/apache-airflow/2.1.0/_images/tree.png | Bin 0 -> 177465 bytes
.../2.1.0/_images/ui-timezone-chooser.png | Bin 0 -> 16644 bytes
.../2.1.0/_images/usage_cli_export.png | Bin 0 -> 397830 bytes
.../2.1.0/_images/usage_cli_imgcat.png | Bin 0 -> 475224 bytes
.../2.1.0/_images/variable_hidden.png | Bin 0 -> 55331 bytes
.../2.1.0/_images/variable_hidden1.png | Bin 0 -> 55331 bytes
.../example_dags/example_bash_operator.html} | 370 +-
.../example_branch_datetime_operator.html} | 373 +-
.../example_branch_day_of_week_operator.html} | 343 +-
.../example_dags/example_branch_labels.html} | 330 +-
.../example_dags/example_dag_decorator.html} | 359 +-
.../example_external_task_marker_dag.html} | 370 +-
.../example_kubernetes_executor_config.html} | 539 +-
.../example_latest_only_with_trigger.html} | 336 +-
.../example_dags/example_python_operator.html} | 385 +-
.../airflow/example_dags/subdags/subdag.html} | 344 +-
.../_modules/airflow/example_dags/tutorial.html} | 419 +-
.../airflow/example_dags/tutorial_etl_dag.html} | 430 +-
.../example_dags/tutorial_taskflow_api_etl.html} | 399 +-
.../tutorial_taskflow_api_etl_virtualenv.html} | 403 +-
.../2.1.0/_modules/airflow/exceptions.html} | 590 +-
.../_modules/airflow/executors/base_executor.html} | 716 +-
.../airflow/executors/celery_executor.html | 1461 +++
.../executors/celery_kubernetes_executor.html} | 564 +-
.../_modules/airflow/executors/dask_executor.html} | 526 +-
.../airflow/executors/debug_executor.html} | 512 +-
.../airflow/executors/executor_constants.html} | 314 +-
.../airflow/executors/executor_loader.html} | 530 +-
.../airflow/executors/kubernetes_executor.html | 1621 +++
.../airflow/executors/local_executor.html} | 757 +-
.../airflow/executors/sequential_executor.html} | 365 +-
.../2.1.0/_modules/airflow/hooks/base.html} | 539 +-
.../2.1.0/_modules/airflow/hooks/dbapi.html} | 756 +-
.../2.1.0/_modules/airflow/hooks/filesystem.html} | 341 +-
.../2.1.0/_modules/airflow/hooks/subprocess.html} | 625 +-
.../2.1.0/_modules/airflow/macros.html} | 373 +-
.../2.1.0/_modules/airflow/macros/hive.html} | 484 +-
.../2.1.0/_modules/airflow/models.html} | 327 +-
.../2.1.0/_modules/airflow/models/base.html} | 336 +-
.../_modules/airflow/models/baseoperator.html | 2512 +++++
.../2.1.0/_modules/airflow/models/connection.html} | 744 +-
.../2.1.0/_modules/airflow/models/crypto.html} | 384 +-
.../2.1.0/_modules/airflow/models/dag.html | 3260 ++++++
.../2.1.0/_modules/airflow/models/dagbag.html | 1493 +++
.../2.1.0/_modules/airflow/models/dagcode.html} | 565 +-
.../2.1.0/_modules/airflow/models/dagparam.html} | 351 +-
.../2.1.0/_modules/airflow/models/dagpickle.html} | 343 +-
.../2.1.0/_modules/airflow/models/dagrun.html | 1642 +++
.../2.1.0/_modules/airflow/models/errors.html} | 327 +-
.../2.1.0/_modules/airflow/models/log.html} | 355 +-
.../2.1.0/_modules/airflow/models/pool.html} | 625 +-
.../_modules/airflow/models/renderedtifields.html} | 558 +-
.../_modules/airflow/models/sensorinstance.html} | 574 +-
.../_modules/airflow/models/serialized_dag.html} | 859 +-
.../2.1.0/_modules/airflow/models/skipmixin.html} | 567 +-
.../2.1.0/_modules/airflow/models/slamiss.html} | 337 +-
.../2.1.0/_modules/airflow/models/taskfail.html} | 341 +-
.../_modules/airflow/models/taskinstance.html | 3014 +++++
.../2.1.0/_modules/airflow/models/taskmixin.html} | 367 +-
.../_modules/airflow/models/taskreschedule.html} | 504 +-
.../2.1.0/_modules/airflow/models/variable.html} | 613 +-
.../2.1.0/_modules/airflow/models/xcom.html} | 639 +-
.../2.1.0/_modules/airflow/models/xcom_arg.html} | 514 +-
.../2.1.0/_modules/airflow/operators/bash.html} | 718 +-
.../2.1.0/_modules/airflow/operators/branch.html} | 344 +-
.../airflow/operators/check_operator.html} | 386 +-
.../_modules/airflow/operators/datetime.html} | 471 +-
.../2.1.0/_modules/airflow/operators/dummy.html} | 327 +-
.../2.1.0/_modules/airflow/operators/email.html} | 379 +-
.../airflow/operators/generic_transfer.html} | 375 +-
.../operators/google_api_to_s3_transfer.html} | 340 +-
.../_modules/airflow/operators/hive_to_druid.html} | 340 +-
.../_modules/airflow/operators/hive_to_mysql.html} | 340 +-
.../_modules/airflow/operators/latest_only.html} | 357 +-
.../_modules/airflow/operators/mssql_to_hive.html} | 339 +-
.../_modules/airflow/operators/mysql_to_hive.html} | 334 +-
.../airflow/operators/presto_check_operator.html} | 369 +-
.../airflow/operators/presto_to_mysql.html} | 340 +-
.../2.1.0/_modules/airflow/operators/python.html} | 875 +-
.../operators/redshift_to_s3_operator.html} | 338 +-
.../airflow/operators/s3_to_hive_operator.html} | 334 +-
.../operators/s3_to_redshift_operator.html} | 339 +-
.../2.1.0/_modules/airflow/operators/sql.html | 1411 +++
.../airflow/operators/sql_branch_operator.html} | 330 +-
.../2.1.0/_modules/airflow/operators/subdag.html} | 576 +-
.../airflow/operators/trigger_dagrun.html} | 537 +-
.../2.1.0/_modules/airflow/operators/weekday.html} | 381 +-
.../2.1.0/_modules/airflow/secrets.html} | 322 +-
.../_modules/airflow/secrets/base_secrets.html} | 397 +-
.../airflow/secrets/environment_variables.html} | 337 +-
.../airflow/secrets/local_filesystem.html} | 876 +-
.../2.1.0/_modules/airflow/secrets/metastore.html} | 362 +-
.../2.1.0/_modules/airflow/sensors/base.html} | 691 +-
.../2.1.0/_modules/airflow/sensors/bash.html} | 374 +-
.../2.1.0/_modules/airflow/sensors/date_time.html} | 361 +-
.../_modules/airflow/sensors/external_task.html} | 667 +-
.../_modules/airflow/sensors/filesystem.html} | 357 +-
.../2.1.0/_modules/airflow/sensors/python.html} | 362 +-
.../_modules/airflow/sensors/smart_sensor.html | 1617 +++
.../2.1.0/_modules/airflow/sensors/sql.html} | 521 +-
.../_modules/airflow/sensors/time_delta.html} | 333 +-
.../_modules/airflow/sensors/time_sensor.html} | 327 +-
.../2.1.0/_modules/airflow/sensors/weekday.html} | 392 +-
.../2.1.0/_modules/airflow/www/security.html | 1593 +++
.../2.1.0/_modules}/index.html | 388 +-
.../_sources/_api/airflow/exceptions/index.rst.txt | 297 +
.../airflow/executors/base_executor/index.rst.txt | 228 +
.../executors/celery_executor/index.rst.txt | 252 +
.../celery_kubernetes_executor/index.rst.txt | 143 +
.../airflow/executors/dask_executor/index.rst.txt | 55 +
.../airflow/executors/debug_executor/index.rst.txt | 85 +
.../executors/executor_constants/index.rst.txt | 44 +
.../executors/executor_loader/index.rst.txt | 67 +
.../_sources/_api/airflow/executors/index.rst.txt | 29 +
.../executors/kubernetes_executor/index.rst.txt | 274 +
.../airflow/executors/local_executor/index.rst.txt | 229 +
.../executors/sequential_executor/index.rst.txt | 54 +
.../_api/airflow/hooks/S3_hook/index.rst.txt | 11 +
.../_sources/_api/airflow/hooks/base/index.rst.txt | 159 +
.../_api/airflow/hooks/base_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/dbapi/index.rst.txt | 257 +
.../_api/airflow/hooks/dbapi_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/docker_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/druid_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/filesystem/index.rst.txt | 38 +
.../_api/airflow/hooks/hdfs_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/hive_hooks/index.rst.txt | 11 +
.../_api/airflow/hooks/http_hook/index.rst.txt | 11 +
.../_sources/_api/airflow/hooks/index.rst.txt | 43 +
.../_api/airflow/hooks/jdbc_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/mssql_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/mysql_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/oracle_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/pig_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/postgres_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/presto_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/samba_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/slack_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/sqlite_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/subprocess/index.rst.txt | 44 +
.../_api/airflow/hooks/webhdfs_hook/index.rst.txt | 11 +
.../_api/airflow/hooks/zendesk_hook/index.rst.txt | 11 +
.../_api/airflow/models/base/index.rst.txt | 43 +
.../_api/airflow/models/baseoperator/index.rst.txt | 934 ++
.../_api/airflow/models/connection/index.rst.txt | 237 +
.../_api/airflow/models/crypto/index.rst.txt | 79 +
.../_sources/_api/airflow/models/dag/index.rst.txt | 1184 ++
.../_api/airflow/models/dagbag/index.rst.txt | 204 +
.../_api/airflow/models/dagcode/index.rst.txt | 131 +
.../_api/airflow/models/dagparam/index.rst.txt | 38 +
.../_api/airflow/models/dagpickle/index.rst.txt | 48 +
.../_api/airflow/models/dagrun/index.rst.txt | 367 +
.../_api/airflow/models/errors/index.rst.txt | 42 +
.../_sources/_api/airflow/models/index.rst.txt | 4284 ++++++++
.../_sources/_api/airflow/models/log/index.rst.txt | 66 +
.../_api/airflow/models/pool/index.rst.txt | 162 +
.../airflow/models/renderedtifields/index.rst.txt | 106 +
.../airflow/models/sensorinstance/index.rst.txt | 156 +
.../airflow/models/serialized_dag/index.rst.txt | 234 +
.../_api/airflow/models/skipmixin/index.rst.txt | 66 +
.../_api/airflow/models/slamiss/index.rst.txt | 68 +
.../_api/airflow/models/taskfail/index.rst.txt | 66 +
.../_api/airflow/models/taskinstance/index.rst.txt | 923 ++
.../_api/airflow/models/taskmixin/index.rst.txt | 80 +
.../airflow/models/taskreschedule/index.rst.txt | 103 +
.../_api/airflow/models/variable/index.rst.txt | 154 +
.../_api/airflow/models/xcom/index.rst.txt | 185 +
.../_api/airflow/models/xcom_arg/index.rst.txt | 117 +
.../_api/airflow/operators/bash/index.rst.txt | 148 +
.../airflow/operators/bash_operator/index.rst.txt | 11 +
.../_api/airflow/operators/branch/index.rst.txt | 47 +
.../operators/branch_operator/index.rst.txt | 11 +
.../airflow/operators/check_operator/index.rst.txt | 46 +
.../operators/dagrun_operator/index.rst.txt | 11 +
.../_api/airflow/operators/datetime/index.rst.txt | 44 +
.../operators/docker_operator/index.rst.txt | 11 +
.../operators/druid_check_operator/index.rst.txt | 11 +
.../_api/airflow/operators/dummy/index.rst.txt | 34 +
.../airflow/operators/dummy_operator/index.rst.txt | 11 +
.../_api/airflow/operators/email/index.rst.txt | 60 +
.../airflow/operators/email_operator/index.rst.txt | 11 +
.../_api/airflow/operators/gcs_to_s3/index.rst.txt | 11 +
.../operators/generic_transfer/index.rst.txt | 58 +
.../google_api_to_s3_transfer/index.rst.txt | 25 +
.../airflow/operators/hive_operator/index.rst.txt | 11 +
.../operators/hive_stats_operator/index.rst.txt | 11 +
.../airflow/operators/hive_to_druid/index.rst.txt | 25 +
.../airflow/operators/hive_to_mysql/index.rst.txt | 25 +
.../operators/hive_to_samba_operator/index.rst.txt | 11 +
.../airflow/operators/http_operator/index.rst.txt | 11 +
.../_sources/_api/airflow/operators/index.rst.txt | 68 +
.../airflow/operators/jdbc_operator/index.rst.txt | 11 +
.../airflow/operators/latest_only/index.rst.txt | 39 +
.../operators/latest_only_operator/index.rst.txt | 11 +
.../airflow/operators/mssql_operator/index.rst.txt | 11 +
.../airflow/operators/mssql_to_hive/index.rst.txt | 25 +
.../airflow/operators/mysql_operator/index.rst.txt | 11 +
.../airflow/operators/mysql_to_hive/index.rst.txt | 22 +
.../operators/oracle_operator/index.rst.txt | 11 +
.../operators/papermill_operator/index.rst.txt | 11 +
.../airflow/operators/pig_operator/index.rst.txt | 11 +
.../operators/postgres_operator/index.rst.txt | 11 +
.../operators/presto_check_operator/index.rst.txt | 38 +
.../operators/presto_to_mysql/index.rst.txt | 25 +
.../_api/airflow/operators/python/index.rst.txt | 280 +
.../operators/python_operator/index.rst.txt | 11 +
.../redshift_to_s3_operator/index.rst.txt | 23 +
.../s3_file_transform_operator/index.rst.txt | 12 +
.../operators/s3_to_hive_operator/index.rst.txt | 22 +
.../s3_to_redshift_operator/index.rst.txt | 25 +
.../airflow/operators/slack_operator/index.rst.txt | 11 +
.../_api/airflow/operators/sql/index.rst.txt | 316 +
.../operators/sql_branch_operator/index.rst.txt | 22 +
.../operators/sqlite_operator/index.rst.txt | 11 +
.../_api/airflow/operators/subdag/index.rst.txt | 116 +
.../operators/subdag_operator/index.rst.txt | 11 +
.../airflow/operators/trigger_dagrun/index.rst.txt | 81 +
.../_api/airflow/operators/weekday/index.rst.txt | 42 +
.../airflow/secrets/base_secrets/index.rst.txt | 81 +
.../secrets/environment_variables/index.rst.txt | 47 +
.../_sources/_api/airflow/secrets/index.rst.txt | 107 +
.../airflow/secrets/local_filesystem/index.rst.txt | 130 +
.../_api/airflow/secrets/metastore/index.rst.txt | 42 +
.../_api/airflow/sensors/base/index.rst.txt | 157 +
.../sensors/base_sensor_operator/index.rst.txt | 11 +
.../_api/airflow/sensors/bash/index.rst.txt | 42 +
.../_api/airflow/sensors/date_time/index.rst.txt | 51 +
.../airflow/sensors/date_time_sensor/index.rst.txt | 11 +
.../airflow/sensors/external_task/index.rst.txt | 157 +
.../sensors/external_task_sensor/index.rst.txt | 11 +
.../_api/airflow/sensors/filesystem/index.rst.txt | 41 +
.../_api/airflow/sensors/hdfs_sensor/index.rst.txt | 11 +
.../sensors/hive_partition_sensor/index.rst.txt | 11 +
.../_api/airflow/sensors/http_sensor/index.rst.txt | 11 +
.../_sources/_api/airflow/sensors/index.rst.txt | 43 +
.../metastore_partition_sensor/index.rst.txt | 12 +
.../named_hive_partition_sensor/index.rst.txt | 11 +
.../_api/airflow/sensors/python/index.rst.txt | 45 +
.../airflow/sensors/s3_key_sensor/index.rst.txt | 11 +
.../airflow/sensors/s3_prefix_sensor/index.rst.txt | 11 +
.../airflow/sensors/smart_sensor/index.rst.txt | 331 +
.../_api/airflow/sensors/sql/index.rst.txt | 63 +
.../_api/airflow/sensors/sql_sensor/index.rst.txt | 11 +
.../_api/airflow/sensors/time_delta/index.rst.txt | 27 +
.../sensors/time_delta_sensor/index.rst.txt | 11 +
.../_api/airflow/sensors/time_sensor/index.rst.txt | 24 +
.../airflow/sensors/web_hdfs_sensor/index.rst.txt | 11 +
.../_api/airflow/sensors/weekday/index.rst.txt | 66 +
.../2.1.0/_sources/backport-providers.rst.txt | 115 +
.../2.1.0/_sources/best-practices.rst.txt | 274 +
.../2.1.0/_sources/changelog.rst.txt | 23 +
.../_sources/cli-and-env-variables-ref.rst.txt | 105 +
.../_sources/concepts/cluster-policies.rst.txt | 92 +
.../2.1.0/_sources/concepts/connections.rst.txt | 37 +
.../2.1.0/_sources/concepts/dags.rst.txt | 608 ++
.../2.1.0/_sources/concepts/index.rst.txt | 55 +
.../2.1.0/_sources/concepts/operators.rst.txt | 201 +
.../2.1.0/_sources/concepts/overview.rst.txt | 96 +
.../2.1.0/_sources/concepts/pools.rst.txt | 52 +
.../_sources/concepts/priority-weight.rst.txt | 61 +
.../2.1.0/_sources/concepts/scheduler.rst.txt | 199 +
.../2.1.0/_sources/concepts/sensors.rst.txt | 33 +
.../2.1.0/_sources/concepts/smart-sensors.rst.txt | 92 +
.../2.1.0/_sources/concepts/taskflow.rst.txt | 72 +
.../2.1.0/_sources/concepts/tasks.rst.txt | 167 +
.../2.1.0/_sources/concepts/variables.rst.txt | 48 +
.../2.1.0/_sources/concepts/xcoms.rst.txt | 49 +
.../2.1.0/_sources/configurations-ref.rst.txt | 96 +
.../apache-airflow/2.1.0/_sources/dag-run.rst.txt | 241 +
.../2.1.0/_sources/dag-serialization.rst.txt | 117 +
.../2.1.0/_sources/deprecated-rest-api-ref.rst.txt | 127 +
.../2.1.0/_sources/executor/celery.rst.txt | 219 +
.../_sources/executor/celery_kubernetes.rst.txt | 46 +
.../2.1.0/_sources/executor/dask.rst.txt | 54 +
.../2.1.0/_sources/executor/debug.rst.txt | 54 +
.../2.1.0/_sources/executor/index.rst.txt | 79 +
.../2.1.0/_sources/executor/kubernetes.rst.txt | 213 +
.../2.1.0/_sources/executor/local.rst.txt | 49 +
.../2.1.0/_sources/executor/sequential.rst.txt | 26 +
.../2.1.0/_sources/extra-packages-ref.rst.txt | 354 +
.../apache-airflow/2.1.0/_sources/faq.rst.txt | 398 +
.../2.1.0/_sources/howto/add-dag-tags.rst.txt | 44 +
.../2.1.0/_sources/howto/connection.rst.txt | 386 +
.../2.1.0/_sources/howto/custom-operator.rst.txt | 270 +
.../customize-dag-ui-page-instance-name.rst.txt | 55 +
.../howto/customize-state-colors-ui.rst.txt | 75 +
.../2.1.0/_sources/howto/define_extra_link.rst.txt | 158 +
.../2.1.0/_sources/howto/email-config.rst.txt | 109 +
.../2.1.0/_sources/howto/index.rst.txt | 46 +
.../2.1.0/_sources/howto/operator/bash.rst.txt | 144 +
.../2.1.0/_sources/howto/operator/datetime.rst.txt | 39 +
.../howto/operator/external_task_sensor.rst.txt | 68 +
.../2.1.0/_sources/howto/operator/index.rst.txt | 38 +
.../2.1.0/_sources/howto/operator/python.rst.txt | 86 +
.../2.1.0/_sources/howto/operator/weekday.rst.txt | 31 +
.../2.1.0/_sources/howto/run-behind-proxy.rst.txt | 92 +
.../2.1.0/_sources/howto/run-with-systemd.rst.txt | 40 +
.../2.1.0/_sources/howto/run-with-upstart.rst.txt | 44 +
.../2.1.0/_sources/howto/set-config.rst.txt | 117 +
.../2.1.0/_sources/howto/set-up-database.rst.txt | 243 +
.../2.1.0/_sources/howto/use-test-config.rst.txt | 36 +
.../2.1.0/_sources/howto/variable.rst.txt | 74 +
.../apache-airflow/2.1.0/_sources/index.rst.txt | 119 +
.../2.1.0/_sources/installation.rst.txt | 354 +
.../2.1.0/_sources/integration.rst.txt | 36 +
.../2.1.0/_sources/kubernetes.rst.txt | 52 +
.../apache-airflow/2.1.0/_sources/license.rst.txt | 201 +
.../apache-airflow/2.1.0/_sources/lineage.rst.txt | 118 +
.../logging-monitoring/check-health.rst.txt | 124 +
.../_sources/logging-monitoring/errors.rst.txt | 72 +
.../_sources/logging-monitoring/index.rst.txt | 38 +
.../logging-architecture.rst.txt | 41 +
.../logging-monitoring/logging-tasks.rst.txt | 109 +
.../_sources/logging-monitoring/metrics.rst.txt | 164 +
.../tracking-user-activity.rst.txt | 39 +
.../2.1.0/_sources/macros-ref.rst.txt | 128 +
.../2.1.0/_sources/modules_management.rst.txt | 276 +
.../2.1.0/_sources/operators-and-hooks-ref.rst.txt | 130 +
.../apache-airflow/2.1.0/_sources/plugins.rst.txt | 349 +
.../2.1.0/_sources/privacy_notice.rst.txt | 68 +
.../2.1.0/_sources/production-deployment.rst.txt | 224 +
.../apache-airflow/2.1.0/_sources/project.rst.txt | 97 +
.../2.1.0/_sources/python-api-ref.rst.txt | 139 +
.../2.1.0/_sources/release-process.rst.txt | 83 +
.../2.1.0/_sources/security/access-control.rst.txt | 243 +
.../2.1.0/_sources/security/api.rst.txt | 163 +
.../2.1.0/_sources/security/flower.rst.txt | 44 +
.../2.1.0/_sources/security/index.rst.txt | 30 +
.../2.1.0/_sources/security/kerberos.rst.txt | 137 +
.../2.1.0/_sources/security/secrets/fernet.rst.txt | 62 +
.../2.1.0/_sources/security/secrets/index.rst.txt | 98 +
.../security/secrets/secrets-backend/index.rst.txt | 97 +
.../local-filesystem-secrets-backend.rst.txt | 145 +
.../2.1.0/_sources/security/webserver.rst.txt | 167 +
.../2.1.0/_sources/security/workload.rst.txt | 56 +
.../2.1.0/_sources/stable-rest-api-ref.rst.txt | 23 +
.../2.1.0/_sources/start/docker.rst.txt | 259 +
.../2.1.0/_sources/start/index.rst.txt | 27 +
.../2.1.0/_sources/start/local.rst.txt | 105 +
.../apache-airflow/2.1.0/_sources/timezone.rst.txt | 183 +
.../apache-airflow/2.1.0/_sources/tutorial.rst.txt | 378 +
.../2.1.0/_sources/tutorial_taskflow_api.rst.txt | 239 +
.../apache-airflow/2.1.0/_sources/ui.rst.txt | 142 +
.../2.1.0/_sources/upgrade-check.rst.txt | 218 +
.../2.1.0/_sources/upgrading-to-2.rst.txt | 1214 +++
.../2.1.0/_sources/usage-cli.rst.txt | 201 +
docs-archive/apache-airflow/2.1.0/_specs/v1.yaml | 3447 ++++++
.../2.1.0/_static/_gen/css/main-custom.min.css | 1 +
.../2.1.0/_static/_gen/css/main.min.css | 7 +
.../apache-airflow/2.1.0/_static/_gen/js/docs.js | 1 +
.../apache-airflow/2.1.0/_static/basic.css | 856 ++
.../apache-airflow/2.1.0/_static/clipboard.min.js | 7 +
.../apache-airflow/2.1.0/_static/copy-button.svg | 5 +
.../apache-airflow/2.1.0/_static/copybutton.css | 67 +
.../apache-airflow/2.1.0/_static/copybutton.js | 153 +
.../2.1.0/_static/copybutton_funcs.js | 47 +
.../apache-airflow/2.1.0/_static/doctools.js | 316 +
.../2.1.0/_static/documentation_options.js | 12 +
.../2.1.0/_static/exampleinclude.css | 86 +
docs-archive/apache-airflow/2.1.0/_static/file.png | Bin 0 -> 286 bytes
.../apache-airflow/2.1.0/_static/graphviz.css | 19 +
.../apache-airflow/2.1.0/_static/jira-links.js | 34 +
.../apache-airflow/2.1.0/_static/jquery-3.5.1.js | 10872 +++++++++++++++++++
.../apache-airflow/2.1.0/_static/jquery.js | 2 +
.../apache-airflow/2.1.0/_static/language_data.js | 297 +
.../apache-airflow/2.1.0/_static/minus.png | Bin 0 -> 90 bytes
.../apache-airflow/2.1.0/_static/pin_32.png | Bin 0 -> 1201 bytes
docs-archive/apache-airflow/2.1.0/_static/plus.png | Bin 0 -> 90 bytes
.../apache-airflow/2.1.0/_static/pygments.css | 74 +
docs-archive/apache-airflow/2.1.0/_static/redoc.js | 103 +
.../apache-airflow/2.1.0/_static/searchtools.js | 514 +
.../2.1.0/_static/underscore-1.3.1.js | 999 ++
.../apache-airflow/2.1.0/_static/underscore.js | 31 +
docs-archive/apache-airflow/2.1.0/airflow.sh | 28 +
.../2.1.0/backport-providers.html} | 332 +-
.../2.1.0/best-practices.html} | 556 +-
docs-archive/apache-airflow/2.1.0/changelog.html | 6071 +++++++++++
.../2.1.0/cli-and-env-variables-ref.html | 4762 ++++++++
docs-archive/apache-airflow/2.1.0/cli-ref.html | 1 +
docs-archive/apache-airflow/2.1.0/concepts.html | 1 +
.../2.1.0/concepts/cluster-policies.html} | 504 +-
.../2.1.0/concepts/connections.html} | 328 +-
.../apache-airflow/2.1.0/concepts/dags.html | 1625 +++
.../2.1.0/concepts}/index.html | 391 +-
.../2.1.0/concepts/operators.html} | 552 +-
.../2.1.0/concepts/overview.html} | 416 +-
.../2.1.0/concepts/pools.html} | 336 +-
.../2.1.0/concepts/priority-weight.html} | 349 +-
.../2.1.0/concepts/scheduler.html} | 546 +-
.../2.1.0/concepts/sensors.html} | 325 +-
.../2.1.0/concepts/smart-sensors.html} | 379 +-
.../2.1.0/concepts/taskflow.html} | 362 +-
.../2.1.0/concepts/tasks.html} | 512 +-
.../2.1.0/concepts/variables.html} | 336 +-
.../2.1.0/concepts/xcoms.html} | 335 +-
.../apache-airflow/2.1.0/configurations-ref.html | 6881 ++++++++++++
.../2.1.0/dag-run.html} | 553 +-
.../2.1.0/dag-serialization.html} | 337 +-
.../2.1.0/deprecated-rest-api-ref.html} | 577 +-
.../apache-airflow/2.1.0/docker-compose.yaml | 147 +
docs-archive/apache-airflow/2.1.0/errors.html | 1 +
.../2.1.0/executor/celery.html} | 553 +-
.../2.1.0/executor/celery_kubernetes.html} | 326 +-
.../2.1.0/executor/dask.html} | 326 +-
.../2.1.0/executor/debug.html} | 328 +-
.../2.1.0/executor}/index.html | 343 +-
.../2.1.0/executor/kubernetes.html} | 771 +-
.../2.1.0/executor/local.html} | 330 +-
.../2.1.0/executor/sequential.html} | 308 +-
.../2.1.0/extra-packages-ref.html} | 983 +-
.../2.1.0/faq.html} | 712 +-
docs-archive/apache-airflow/2.1.0/genindex.html | 6873 ++++++++++++
.../2.1.0/howto/add-dag-tags.html} | 325 +-
.../apache-airflow/2.1.0/howto/add-new-role.html | 1 +
.../apache-airflow/2.1.0/howto/check-health.html | 1 +
.../2.1.0/howto/connection.html} | 710 +-
.../2.1.0/howto/connection/index.html | 1 +
.../2.1.0/howto/custom-operator.html} | 602 +-
.../customize-dag-ui-page-instance-name.html} | 355 +-
.../2.1.0/howto/customize-state-colors-ui.html} | 372 +-
.../2.1.0/howto/define_extra_link.html} | 508 +-
.../2.1.0/howto/email-config.html} | 429 +-
.../2.1.0/howto}/index.html | 382 +-
.../2.1.0/howto/initialize-database.html | 1 +
.../2.1.0/howto/operator/bash.html} | 521 +-
.../2.1.0/howto/operator/datetime.html} | 410 +-
.../2.1.0/howto/operator/external.html | 1 +
.../howto/operator/external_task_sensor.html} | 426 +-
.../2.1.0/howto/operator}/index.html | 358 +-
.../2.1.0/howto/operator/python.html} | 477 +-
.../2.1.0/howto/operator/weekday.html} | 345 +-
.../2.1.0/howto/run-behind-proxy.html} | 381 +-
.../2.1.0/howto/run-with-systemd.html} | 327 +-
.../2.1.0/howto/run-with-upstart.html} | 329 +-
.../2.1.0/howto/secure-connections.html | 1 +
.../2.1.0/howto/set-config.html} | 614 +-
.../2.1.0/howto/set-up-database.html} | 569 +-
.../2.1.0/howto/tracking-user-activity.html | 1 +
.../howto/use-alternative-secrets-backend.html | 1 +
.../2.1.0/howto/use-test-config.html} | 322 +-
.../2.1.0/howto/variable.html} | 361 +-
.../apache-airflow/2.1.0/howto/write-logs.html | 1 +
.../2.1.0/http-routingtable.html} | 344 +-
.../2.1.0}/index.html | 269 +-
.../2.1.0/installation.html} | 622 +-
.../2.1.0/integration.html} | 263 +-
.../2.1.0/kubernetes.html} | 279 +-
.../2.1.0/license.html} | 516 +-
.../2.1.0/lineage.html} | 340 +-
.../2.1.0/logging-monitoring/check-health.html} | 426 +-
.../2.1.0/logging-monitoring/errors.html} | 379 +-
.../2.1.0/logging-monitoring}/index.html | 302 +-
.../logging-monitoring/logging-architecture.html} | 306 +-
.../2.1.0/logging-monitoring/logging-tasks.html} | 384 +-
.../2.1.0/logging-monitoring/metrics.html} | 668 +-
.../tracking-user-activity.html} | 311 +-
.../2.1.0/macros-ref.html} | 715 +-
docs-archive/apache-airflow/2.1.0/metrics.html | 1 +
.../2.1.0/modules_management.html} | 528 +-
docs-archive/apache-airflow/2.1.0/objects.inv | Bin 0 -> 26839 bytes
.../2.1.0/operators-and-hooks-ref.html} | 385 +-
.../2.1.0/output-build-apache-airflow.log | 1519 +++
.../2.1.0/output-spelling-apache-airflow.log | 1519 +++
.../_static/redoc.js | 8 +
.../2.1.0/plugins.html} | 630 +-
.../2.1.0/privacy_notice.html} | 287 +-
.../2.1.0/production-deployment.html} | 494 +-
.../2.1.0/project.html} | 326 +-
.../2.1.0/py-modindex.html} | 1067 +-
.../2.1.0/python-api-ref.html} | 502 +-
.../2.1.0/release-process.html} | 298 +-
.../apache-airflow/2.1.0/rest-api-ref.html | 1 +
docs-archive/apache-airflow/2.1.0/scheduler.html | 1 +
.../2.1.0/search.html} | 281 +-
docs-archive/apache-airflow/2.1.0/searchindex.js | 1 +
docs-archive/apache-airflow/2.1.0/security.html | 1 +
.../2.1.0/security/access-control.html} | 983 +-
.../2.1.0/security/api.html} | 504 +-
.../2.1.0/security/flower.html} | 321 +-
.../2.1.0/security}/index.html | 322 +-
.../2.1.0/security/kerberos.html} | 415 +-
.../2.1.0/security/secrets/fernet.html} | 353 +-
.../2.1.0/security/secrets}/index.html | 375 +-
.../security/secrets/secrets-backend/index.html} | 612 +-
.../local-filesystem-secrets-backend.html} | 511 +-
.../2.1.0/security/webserver.html} | 476 +-
.../2.1.0/security/workload.html} | 340 +-
.../apache-airflow/2.1.0/smart-sensor.html | 1 +
.../apache-airflow/2.1.0/stable-rest-api-ref.html | 34 +
docs-archive/apache-airflow/2.1.0/start.html | 1 +
.../2.1.0/start/docker.html} | 581 +-
.../2.1.0/start}/index.html | 288 +-
.../2.1.0/start/local.html} | 362 +-
.../2.1.0/timezone.html} | 477 +-
.../2.1.0/tutorial.html} | 925 +-
.../2.1.0/tutorial_taskflow_api.html} | 737 +-
.../index.html => apache-airflow/2.1.0/ui.html} | 354 +-
.../2.1.0/upgrade-check.html} | 480 +-
.../apache-airflow/2.1.0/upgrading-to-2.html | 2270 ++++
.../2.1.0/usage-cli.html} | 468 +-
.../2.1.0/warning-build-apache-airflow.log | 0
docs-archive/apache-airflow/stable.txt | 2 +-
docs-archive/docker-stack/.buildinfo | 2 +-
.../extending/embedding-dags/test_dag.html} | 253 +-
.../docker-stack/_sources/build-arg-ref.rst.txt | 16 +-
docs-archive/docker-stack/_sources/build.rst.txt | 18 +-
.../docker-stack/_sources/entrypoint.rst.txt | 24 +-
docs-archive/docker-stack/_sources/recipes.rst.txt | 4 +-
docs-archive/docker-stack/build-arg-ref.html | 20 +-
docs-archive/docker-stack/build.html | 10 +-
docs-archive/docker-stack/entrypoint.html | 19 +-
docs-archive/docker-stack/index.html | 4 +-
.../docker-stack/output-build-docker-stack.log | 122 -
.../docker-stack/output-spelling-docker-stack.log | 118 -
docs-archive/docker-stack/recipes.html | 1 +
docs-archive/docker-stack/searchindex.js | 2 +-
713 files changed, 218242 insertions(+), 67668 deletions(-)
diff --git a/docs-archive/docker-stack/.buildinfo b/docs-archive/apache-airflow/2.1.0/.buildinfo
similarity index 82%
copy from docs-archive/docker-stack/.buildinfo
copy to docs-archive/apache-airflow/2.1.0/.buildinfo
index 21af09a..7a0d7ba 100644
--- a/docs-archive/docker-stack/.buildinfo
+++ b/docs-archive/apache-airflow/2.1.0/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: a8e7ce1a9dc241634134293c12d40694
+config: 05c85217c07acdbfae531a47e1b29f33
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/docs-archive/docker-stack/entrypoint.html b/docs-archive/apache-airflow/2.1.0/_api/airflow/exceptions/index.html
similarity index 55%
copy from docs-archive/docker-stack/entrypoint.html
copy to docs-archive/apache-airflow/2.1.0/_api/airflow/exceptions/index.html
index aee6e73..34d6e60 100644
--- a/docs-archive/docker-stack/entrypoint.html
+++ b/docs-archive/apache-airflow/2.1.0/_api/airflow/exceptions/index.html
@@ -9,15 +9,16 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Entrypoint — docker-stack Documentation</title>
- <link rel="stylesheet" href="_static/_gen/css/main.min.css" type="text/css" />
- <link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
- <link rel="stylesheet" type="text/css" href="_static/_gen/css/main-custom.min.css" />
- <link rel="shortcut icon" href="_static/pin_32.png"/>
- <link rel="index" title="Index" href="genindex.html" />
- <link rel="search" title="Search" href="search.html" />
- <link rel="next" title="Recipes" href="recipes.html" />
- <link rel="prev" title="Building the image" href="build.html" />
+ <title>airflow.exceptions — Airflow Documentation</title>
+ <link rel="stylesheet" href="../../../_static/_gen/css/main.min.css" type="text/css" />
+ <link rel="stylesheet" type="text/css" href="../../../_static/copybutton.css" />
+ <link rel="stylesheet" type="text/css" href="../../../_static/_gen/css/main-custom.min.css" />
+ <link rel="stylesheet" type="text/css" href="../../../_static/graphviz.css" />
+ <link rel="shortcut icon" href="../../../_static/pin_32.png"/>
+ <link rel="index" title="Index" href="../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../search.html" />
+ <link rel="next" title="airflow.secrets" href="../secrets/index.html" />
+ <link rel="prev" title="airflow.models.xcom_arg" href="../models/xcom_arg/index.html" />
<script type="application/javascript">
var doNotTrack = false;
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
@@ -272,7 +273,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -283,7 +284,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -313,22 +314,187 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Entrypoint</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#allowing-arbitrary-user-to-run-the-container">Allowing arbitrary user to run the container</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#waits-for-airflow-db-connection">Waits for Airflow DB connection</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#upgrading-airflow-db">Upgrading Airflow DB</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#creating-admin-user">Creating admin user</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#waits-for-celery-broker-connection">Waits for celery broker connection</a></li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../privacy_notice.html">Privacy Notice</a></li>
+</ul>
+<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#executors">Executors</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../python-api-ref.html#exceptions">Exceptions</a><ul class="current">
+<li class="toctree-l3 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.exceptions</span></code></a><ul>
+<li class="toctree-l4"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowException">AirflowException</a><ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.AirflowException.status_code">status_code</a></li>
</ul>
</li>
-<li class="toctree-l1"><a class="reference internal" href="recipes.html">Recipes</a></li>
</ul>
-<p class="caption"><span class="caption-text">References</span></p>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowBadRequest">AirflowBadRequest</a><ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.AirflowBadRequest.status_code">status_code</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowNotFoundException">AirflowNotFoundException</a><ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.AirflowNotFoundException.status_code">status_code</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowConfigException">AirflowConfigException</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowSensorTimeout">AirflowSensorTimeout</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowRescheduleException">AirflowRescheduleException</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowSmartSensorException">AirflowSmartSensorException</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.InvalidStatsNameException">InvalidStatsNameException</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowTaskTimeout">AirflowTaskTimeout</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowWebServerTimeout">AirflowWebServerTimeout</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowSkipException">AirflowSkipException</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowFailException">AirflowFailException</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowDagCycleException">AirflowDagCycleException</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowDagDuplicatedIdException">AirflowDagDuplicatedIdException</a><ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.AirflowDagDuplicatedIdException.__str__">__str__</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowClusterPolicyViolation">AirflowClusterPolicyViolation</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.DagNotFound">DagNotFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.DagCodeNotFound">DagCodeNotFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.DagRunNotFound">DagRunNotFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.DagRunAlreadyExists">DagRunAlreadyExists</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.DagFileExists">DagFileExists</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.DuplicateTaskIdFound">DuplicateTaskIdFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.SerializedDagNotFound">SerializedDagNotFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.SerializationError">SerializationError</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.TaskNotFound">TaskNotFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.TaskInstanceNotFound">TaskInstanceNotFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.PoolNotFound">PoolNotFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.NoAvailablePoolSlot">NoAvailablePoolSlot</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.DagConcurrencyLimitReached">DagConcurrencyLimitReached</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.TaskConcurrencyLimitReached">TaskConcurrencyLimitReached</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.BackfillUnfinished">BackfillUnfinished</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.FileSyntaxError">FileSyntaxError</a><ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.FileSyntaxError.line_no">line_no</a></li>
+</ul>
+<ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.FileSyntaxError.message">message</a></li>
+</ul>
+<ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.FileSyntaxError.__str__">__str__</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowFileParseException">AirflowFileParseException</a><ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.AirflowFileParseException.__str__">__str__</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.ConnectionNotUnique">ConnectionNotUnique</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -412,7 +578,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -423,7 +589,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -453,22 +619,187 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Entrypoint</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#allowing-arbitrary-user-to-run-the-container">Allowing arbitrary user to run the container</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#waits-for-airflow-db-connection">Waits for Airflow DB connection</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#upgrading-airflow-db">Upgrading Airflow DB</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#creating-admin-user">Creating admin user</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#waits-for-celery-broker-connection">Waits for celery broker connection</a></li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../privacy_notice.html">Privacy Notice</a></li>
+</ul>
+<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#executors">Executors</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../python-api-ref.html#exceptions">Exceptions</a><ul class="current">
+<li class="toctree-l3 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.exceptions</span></code></a><ul>
+<li class="toctree-l4"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowException">AirflowException</a><ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.AirflowException.status_code">status_code</a></li>
</ul>
</li>
-<li class="toctree-l1"><a class="reference internal" href="recipes.html">Recipes</a></li>
</ul>
-<p class="caption"><span class="caption-text">References</span></p>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowBadRequest">AirflowBadRequest</a><ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.AirflowBadRequest.status_code">status_code</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowNotFoundException">AirflowNotFoundException</a><ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.AirflowNotFoundException.status_code">status_code</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowConfigException">AirflowConfigException</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowSensorTimeout">AirflowSensorTimeout</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowRescheduleException">AirflowRescheduleException</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowSmartSensorException">AirflowSmartSensorException</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.InvalidStatsNameException">InvalidStatsNameException</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowTaskTimeout">AirflowTaskTimeout</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowWebServerTimeout">AirflowWebServerTimeout</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowSkipException">AirflowSkipException</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowFailException">AirflowFailException</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowDagCycleException">AirflowDagCycleException</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowDagDuplicatedIdException">AirflowDagDuplicatedIdException</a><ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.AirflowDagDuplicatedIdException.__str__">__str__</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowClusterPolicyViolation">AirflowClusterPolicyViolation</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.DagNotFound">DagNotFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.DagCodeNotFound">DagCodeNotFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.DagRunNotFound">DagRunNotFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.DagRunAlreadyExists">DagRunAlreadyExists</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.DagFileExists">DagFileExists</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.DuplicateTaskIdFound">DuplicateTaskIdFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.SerializedDagNotFound">SerializedDagNotFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.SerializationError">SerializationError</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.TaskNotFound">TaskNotFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.TaskInstanceNotFound">TaskInstanceNotFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.PoolNotFound">PoolNotFound</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.NoAvailablePoolSlot">NoAvailablePoolSlot</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.DagConcurrencyLimitReached">DagConcurrencyLimitReached</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.TaskConcurrencyLimitReached">TaskConcurrencyLimitReached</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.BackfillUnfinished">BackfillUnfinished</a></li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.FileSyntaxError">FileSyntaxError</a><ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.FileSyntaxError.line_no">line_no</a></li>
+</ul>
+<ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.FileSyntaxError.message">message</a></li>
+</ul>
+<ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.FileSyntaxError.__str__">__str__</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.AirflowFileParseException">AirflowFileParseException</a><ul>
+<li class="toctree-l6"><a class="reference internal" href="#airflow.exceptions.AirflowFileParseException.__str__">__str__</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li class="toctree-l5"><a class="reference internal" href="#airflow.exceptions.ConnectionNotUnique">ConnectionNotUnique</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -557,9 +888,11 @@
<ul class="breadcrumb">
- <li class="breadcrumb-item"><a href="index.html" class="icon icon-home"> Home</a></li>
+ <li class="breadcrumb-item"><a href="../../../index.html" class="icon icon-home"> Home</a></li>
+
+ <li class="breadcrumb-item"><a href="../../../python-api-ref.html">Python API Reference</a></li>
- <li class="breadcrumb-item"><a href="entrypoint.html"> Entrypoint</a></li>
+ <li class="breadcrumb-item"><a href="index.html"> <code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.exceptions</span></code></a></li>
</ul>
</div>
@@ -570,224 +903,316 @@
<div class="bodywrapper">
<div class="body" role="main">
- <blockquote>
-<div></div></blockquote>
-<div class="section" id="entrypoint">
-<h1>Entrypoint<a class="headerlink" href="#entrypoint" title="Permalink to this headline">¶</a></h1>
-<p>If you are using the default entrypoint of the production image,
-there are a few actions that are automatically performed when the container starts.
-In some cases, you can pass environment variables to the image to trigger some of that behaviour.</p>
-<p>The variables that control the "execution" behaviour start with <code class="docutils literal notranslate"><span class="pre">_AIRFLOW</span></code> to distinguish them
-from the variables used to build the image starting with <code class="docutils literal notranslate"><span class="pre">AIRFLOW</span></code>.</p>
-<p>The image entrypoint works as follows:</p>
-<ul>
-<li><p>In case the user is not "airflow" (with undefined user id) and the group id of the user is set to <code class="docutils literal notranslate"><span class="pre">0</span></code> (root),
-then the user is dynamically added to <code class="docutils literal notranslate"><span class="pre">/etc/passwd</span></code> at entry using <code class="docutils literal notranslate"><span class="pre">USER_NAME</span></code> variable to define the user name.
-This is in order to accommodate the
-<a class="reference external" href="https://docs.openshift.com/enterprise/3.0/creating_images/guidelines.html">OpenShift Guidelines</a></p></li>
-<li><p>The <code class="docutils literal notranslate"><span class="pre">AIRFLOW_HOME</span></code> is set by default to <code class="docutils literal notranslate"><span class="pre">/opt/airflow/</span></code> - this means that DAGs
-are in default in the <code class="docutils literal notranslate"><span class="pre">/opt/airflow/dags</span></code> folder and logs are in the <code class="docutils literal notranslate"><span class="pre">/opt/airflow/logs</span></code></p></li>
-<li><p>The working directory is <code class="docutils literal notranslate"><span class="pre">/opt/airflow</span></code> by default.</p></li>
-<li><p>If <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_CONN</span></code> variable is passed to the container and it is either mysql or postgres
-SQL alchemy connection, then the connection is checked and the script waits until the database is reachable.
-If <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_CONN_CMD</span></code> variable is passed to the container, it is evaluated as a
-command to execute and result of this evaluation is used as <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_CONN</span></code>. The
-<code class="docutils literal notranslate"><span class="pre">_CMD</span></code> variable takes precedence over the <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_CONN</span></code> variable.</p></li>
-<li><p>If no <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_CONN</span></code> variable is set then SQLite database is created in
-<code class="docutils literal notranslate"><span class="pre">${AIRFLOW_HOME}/airflow.db</span></code> and db reset is executed.</p></li>
-<li><p>If first argument equals to "bash" - you are dropped to a bash shell or you can executes bash command
-if you specify extra arguments. For example:</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker run -it apache/airflow:master-python3.6 bash -c <span class="s2">"ls -la"</span>
-total <span class="m">16</span>
-drwxr-xr-x <span class="m">4</span> airflow root <span class="m">4096</span> Jun <span class="m">5</span> <span class="m">18</span>:12 .
-drwxr-xr-x <span class="m">1</span> root root <span class="m">4096</span> Jun <span class="m">5</span> <span class="m">18</span>:12 ..
-drwxr-xr-x <span class="m">2</span> airflow root <span class="m">4096</span> Jun <span class="m">5</span> <span class="m">18</span>:12 dags
-drwxr-xr-x <span class="m">2</span> airflow root <span class="m">4096</span> Jun <span class="m">5</span> <span class="m">18</span>:12 logs
-</pre></div>
-</div>
-</li>
-<li><p>If first argument is equal to <code class="docutils literal notranslate"><span class="pre">python</span></code> - you are dropped in python shell or python commands are executed if
-you pass extra parameters. For example:</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>> docker run -it apache/airflow:master-python3.6 python -c <span class="s2">"print('test')"</span>
-<span class="nb">test</span>
-</pre></div>
-</div>
-</li>
-<li><p>If first argument equals to "airflow" - the rest of the arguments is treated as an airflow command
-to execute. Example:</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker run -it apache/airflow:master-python3.6 airflow webserver
-</pre></div>
-</div>
-</li>
-<li><p>If there are any other arguments - they are simply passed to the "airflow" command</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>> docker run -it apache/airflow:master-python3.6 version
-<span class="m">2</span>.1.0.dev0
-</pre></div>
-</div>
-</li>
-<li><p>If <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__BROKER_URL</span></code> variable is passed and airflow command with
-scheduler, worker of flower command is used, then the script checks the broker connection
-and waits until the Celery broker database is reachable.
-If <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__BROKER_URL_CMD</span></code> variable is passed to the container, it is evaluated as a
-command to execute and result of this evaluation is used as <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__BROKER_URL</span></code>. The
-<code class="docutils literal notranslate"><span class="pre">_CMD</span></code> variable takes precedence over the <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__BROKER_URL</span></code> variable.</p></li>
-</ul>
-<div class="section" id="allowing-arbitrary-user-to-run-the-container">
-<span id="arbitrary-docker-user"></span><h2>Allowing arbitrary user to run the container<a class="headerlink" href="#allowing-arbitrary-user-to-run-the-container" title="Permalink to this headline">¶</a></h2>
-<p>Airflow image is Open-Shift compatible, which means that you can start it with random user ID and the
-group id <code class="docutils literal notranslate"><span class="pre">0</span></code> (<code class="docutils literal notranslate"><span class="pre">root</span></code>). If you want to run the image with user different than Airflow, you MUST set
-GID of the user to <code class="docutils literal notranslate"><span class="pre">0</span></code>. In case you try to use different group, the entrypoint exits with error.</p>
-<p>In order to accommodate a number of external libraries and projects, Airflow will automatically create
-such an arbitrary user in (<cite>/etc/passwd</cite>) and make it's home directory point to <code class="docutils literal notranslate"><span class="pre">/home/airflow</span></code>.
-Many of 3rd-party libraries and packages require home directory of the user to be present, because they
-need to write some cache information there, so such a dynamic creation of a user is necessary.</p>
-<p>Such arbitrary user has to be able to write to certain directories that needs write access, and since
-it is not advised to allow write access to "other" for security reasons, the OpenShift
-guidelines introduced the concept of making all such folders have the <code class="docutils literal notranslate"><span class="pre">0</span></code> (<code class="docutils literal notranslate"><span class="pre">root</span></code>) group id (GID).
-All the directories that need write access in the Airflow production image have GID set to 0 (and
-they are writable for the group). We are following that concept and all the directories that need
-write access follow that.</p>
-<p>The GID=0 is set as default for the <code class="docutils literal notranslate"><span class="pre">airflow</span></code> user, so any directories it creates have GID set to 0
-by default. The entrypoint sets <code class="docutils literal notranslate"><span class="pre">umask</span></code> to be <code class="docutils literal notranslate"><span class="pre">0002</span></code> - this means that any directories created by
-the user have also "group write" access for group <code class="docutils literal notranslate"><span class="pre">0</span></code> - they will be writable by other users with
-<code class="docutils literal notranslate"><span class="pre">root</span></code> group. Also whenever any "arbitrary" user creates a folder (for example in a mounted volume), that
-folder will have a "group write" access and <code class="docutils literal notranslate"><span class="pre">GID=0</span></code>, so that execution with another, arbitrary user
-will still continue to work, even if such directory is mounted by another arbitrary user later.</p>
-<p>The <code class="docutils literal notranslate"><span class="pre">umask</span></code> setting however only works for runtime of the container - it is not used during building of
-the image. If you would like to extend the image and add your own packages, you should remember to add
-<code class="docutils literal notranslate"><span class="pre">umask</span> <span class="pre">0002</span></code> in front of your docker command - this way the directories created by any installation
-that need group access will also be writable for the group. This can be done for example this way:</p>
-<blockquote>
-<div><div class="highlight-docker notranslate"><div class="highlight"><pre><span></span><span class="k">RUN</span> <span class="nb">umask</span> <span class="m">0002</span><span class="p">;</span> <span class="se">\</span>
- do_something<span class="p">;</span> <span class="se">\</span>
- do_otherthing<span class="p">;</span>
-</pre></div>
-</div>
-</div></blockquote>
-<p>You can read more about it in the "Support arbitrary user ids" chapter in the
-<a class="reference external" href="https://docs.openshift.com/container-platform/4.7/openshift_images/create-images.html#images-create-guide-openshift_create-images">Openshift best practices</a>.</p>
-</div>
-<div class="section" id="waits-for-airflow-db-connection">
-<h2>Waits for Airflow DB connection<a class="headerlink" href="#waits-for-airflow-db-connection" title="Permalink to this headline">¶</a></h2>
-<p>In case Postgres or MySQL DB is used, the entrypoint will wait until the airflow DB connection becomes
-available. This happens always when you use the default entrypoint.</p>
-<p>The script detects backend type depending on the URL schema and assigns default port numbers if not specified
-in the URL. Then it loops until the connection to the host/port specified can be established
-It tries <code class="docutils literal notranslate"><span class="pre">CONNECTION_CHECK_MAX_COUNT</span></code> times and sleeps <code class="docutils literal notranslate"><span class="pre">CONNECTION_CHECK_SLEEP_TIME</span></code> between checks
-To disable check, set <code class="docutils literal notranslate"><span class="pre">CONNECTION_CHECK_MAX_COUNT=0</span></code>.</p>
-<p>Supported schemes:</p>
-<ul class="simple">
-<li><p><code class="docutils literal notranslate"><span class="pre">postgres://</span></code> - default port 5432</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">mysql://</span></code> - default port 3306</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">sqlite://</span></code></p></li>
-</ul>
-<p>In case of SQLite backend, there is no connection to establish and waiting is skipped.</p>
-</div>
-<div class="section" id="upgrading-airflow-db">
-<h2>Upgrading Airflow DB<a class="headerlink" href="#upgrading-airflow-db" title="Permalink to this headline">¶</a></h2>
-<p>If you set <code class="docutils literal notranslate"><span class="pre">_AIRFLOW_DB_UPGRADE</span></code> variable to a non-empty value, the entrypoint will run
-the <code class="docutils literal notranslate"><span class="pre">airflow</span> <span class="pre">db</span> <span class="pre">upgrade</span></code> command right after verifying the connection. You can also use this
-when you are running airflow with internal SQLite database (default) to upgrade the db and create
-admin users at entrypoint, so that you can start the webserver immediately. Note - using SQLite is
-intended only for testing purpose, never use SQLite in production as it has severe limitations when it
-comes to concurrency.</p>
-</div>
-<div class="section" id="creating-admin-user">
-<h2>Creating admin user<a class="headerlink" href="#creating-admin-user" title="Permalink to this headline">¶</a></h2>
-<p>The entrypoint can also create webserver user automatically when you enter it. you need to set
-<code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_CREATE</span></code> to a non-empty value in order to do that. This is not intended for
-production, it is only useful if you would like to run a quick test with the production image.
-You need to pass at least password to create such user via <code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_PASSWORD</span></code> or
-<code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_PASSWORD_CMD</span></code> similarly like for other <code class="docutils literal notranslate"><span class="pre">*_CMD</span></code> variables, the content of
-the <code class="docutils literal notranslate"><span class="pre">*_CMD</span></code> will be evaluated as shell command and it's output will be set as password.</p>
-<p>User creation will fail if none of the <code class="docutils literal notranslate"><span class="pre">PASSWORD</span></code> variables are set - there is no default for
-password for security reasons.</p>
-<table class="docutils align-default">
-<colgroup>
-<col style="width: 10%" />
-<col style="width: 24%" />
-<col style="width: 65%" />
-</colgroup>
-<thead>
-<tr class="row-odd"><th class="head"><p>Parameter</p></th>
-<th class="head"><p>Default</p></th>
-<th class="head"><p>Environment variable</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p>username</p></td>
-<td><p>admin</p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_USERNAME</span></code></p></td>
-</tr>
-<tr class="row-odd"><td><p>password</p></td>
-<td></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_PASSWORD_CMD</span></code> or <code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_PASSWORD</span></code></p></td>
-</tr>
-<tr class="row-even"><td><p>firstname</p></td>
-<td><p>Airflow</p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_FIRSTNAME</span></code></p></td>
-</tr>
-<tr class="row-odd"><td><p>lastname</p></td>
-<td><p>Admin</p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_LASTNAME</span></code></p></td>
-</tr>
-<tr class="row-even"><td><p>email</p></td>
-<td><p><a class="reference external" href="mailto:airflowadmin%40example.com">airflowadmin<span>@</span>example<span>.</span>com</a></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_EMAIL</span></code></p></td>
-</tr>
-<tr class="row-odd"><td><p>role</p></td>
-<td><p>Admin</p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_ROLE</span></code></p></td>
-</tr>
-</tbody>
-</table>
-<p>In case the password is specified, the user will be attempted to be created, but the entrypoint will
-not fail if the attempt fails (this accounts for the case that the user is already created).</p>
-<p>You can, for example start the webserver in the production image with initializing the internal SQLite
-database and creating an <code class="docutils literal notranslate"><span class="pre">admin/admin</span></code> Admin user with the following command:</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker run -it -p <span class="m">8080</span>:8080 <span class="se">\</span>
- --env <span class="s2">"_AIRFLOW_DB_UPGRADE=true"</span> <span class="se">\</span>
- --env <span class="s2">"_AIRFLOW_WWW_USER_CREATE=true"</span> <span class="se">\</span>
- --env <span class="s2">"_AIRFLOW_WWW_USER_PASSWORD=admin"</span> <span class="se">\</span>
- apache/airflow:master-python3.8 webserver
-</pre></div>
-</div>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker run -it -p <span class="m">8080</span>:8080 <span class="se">\</span>
- --env <span class="s2">"_AIRFLOW_DB_UPGRADE=true"</span> <span class="se">\</span>
- --env <span class="s2">"_AIRFLOW_WWW_USER_CREATE=true"</span> <span class="se">\</span>
- --env <span class="s2">"_AIRFLOW_WWW_USER_PASSWORD_CMD=echo admin"</span> <span class="se">\</span>
- apache/airflow:master-python3.8 webserver
-</pre></div>
-</div>
-<p>The commands above perform initialization of the SQLite database, create admin user with admin password
-and Admin role. They also forward local port <code class="docutils literal notranslate"><span class="pre">8080</span></code> to the webserver port and finally start the webserver.</p>
-</div>
-<div class="section" id="waits-for-celery-broker-connection">
-<h2>Waits for celery broker connection<a class="headerlink" href="#waits-for-celery-broker-connection" title="Permalink to this headline">¶</a></h2>
-<p>In case Postgres or MySQL DB is used, and one of the <code class="docutils literal notranslate"><span class="pre">scheduler</span></code>, <code class="docutils literal notranslate"><span class="pre">celery</span></code>, <code class="docutils literal notranslate"><span class="pre">worker</span></code>, or <code class="docutils literal notranslate"><span class="pre">flower</span></code>
-commands are used the entrypoint will wait until the celery broker DB connection is available.</p>
-<p>The script detects backend type depending on the URL schema and assigns default port numbers if not specified
-in the URL. Then it loops until connection to the host/port specified can be established
-It tries <code class="docutils literal notranslate"><span class="pre">CONNECTION_CHECK_MAX_COUNT</span></code> times and sleeps <code class="docutils literal notranslate"><span class="pre">CONNECTION_CHECK_SLEEP_TIME</span></code> between checks.
-To disable check, set <code class="docutils literal notranslate"><span class="pre">CONNECTION_CHECK_MAX_COUNT=0</span></code>.</p>
-<p>Supported schemes:</p>
-<ul class="simple">
-<li><p><code class="docutils literal notranslate"><span class="pre">amqp(s)://</span></code> (rabbitmq) - default port 5672</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">redis://</span></code> - default port 6379</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">postgres://</span></code> - default port 5432</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">mysql://</span></code> - default port 3306</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">sqlite://</span></code></p></li>
-</ul>
-<p>In case of SQLite backend, there is no connection to establish and waiting is skipped.</p>
+ <div class="section" id="module-airflow.exceptions">
+<span id="airflow-exceptions"></span><h1><a class="reference internal" href="#module-airflow.exceptions" title="airflow.exceptions"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.exceptions</span></code></a><a class="headerlink" href="#module-airflow.exceptions" title="Permalink to this headline">¶</a></h1>
+<p><p>Exceptions used by Airflow</p>
+</p>
+<div class="section" id="module-contents">
+<h2>Module Contents<a class="headerlink" href="#module-contents" title="Permalink to this headline">¶</a></h2>
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowException">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowException</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowException"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowException" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/exceptions.html#Exception" title="(in Python v3.9)"><code class="xref py py-class docutils literal notranslate"><span class="pre">Exception</span></code></a></p>
+<p>Base class for all Airflow's errors.
+Each custom exception should be derived from this class</p>
+<dl class="py attribute">
+<dt id="airflow.exceptions.AirflowException.status_code">
+<code class="sig-name descname">status_code</code><em class="property"> = 500</em><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowException.status_code"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowException.status_code" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowBadRequest">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowBadRequest</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowBadRequest"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowBadRequest" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when the application or server cannot handle the request</p>
+<dl class="py attribute">
+<dt id="airflow.exceptions.AirflowBadRequest.status_code">
+<code class="sig-name descname">status_code</code><em class="property"> = 400</em><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowBadRequest.status_code"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowBadRequest.status_code" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowNotFoundException">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowNotFoundException</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowNotFoundException"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowNotFoundException" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when the requested object/resource is not available in the system</p>
+<dl class="py attribute">
+<dt id="airflow.exceptions.AirflowNotFoundException.status_code">
+<code class="sig-name descname">status_code</code><em class="property"> = 404</em><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowNotFoundException.status_code"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowNotFoundException.status_code" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowConfigException">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowConfigException</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowConfigException"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowConfigException" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when there is configuration problem</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowSensorTimeout">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowSensorTimeout</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowSensorTimeout"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowSensorTimeout" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when there is a timeout on sensor polling</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowRescheduleException">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowRescheduleException</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">reschedule_date</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowRescheduleException"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.except [...]
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when the task should be re-scheduled at a later time.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>reschedule_date</strong> (<a class="reference external" href="https://docs.python.org/3/library/datetime.html#datetime.datetime" title="(in Python v3.9)"><em>datetime.datetime</em></a>) -- The date when the task should be rescheduled</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowSmartSensorException">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowSmartSensorException</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowSmartSensorException"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowSmartSensorException" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise after the task register itself in the smart sensor service
+It should exit without failing a task</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.InvalidStatsNameException">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">InvalidStatsNameException</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#InvalidStatsNameException"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.InvalidStatsNameException" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when name of the stats is invalid</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowTaskTimeout">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowTaskTimeout</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowTaskTimeout"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowTaskTimeout" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when the task execution times-out</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowWebServerTimeout">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowWebServerTimeout</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowWebServerTimeout"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowWebServerTimeout" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when the web server times out</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowSkipException">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowSkipException</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowSkipException"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowSkipException" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when the task should be skipped</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowFailException">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowFailException</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowFailException"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowFailException" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when the task should be failed without retrying</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowDagCycleException">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowDagCycleException</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowDagCycleException"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowDagCycleException" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when there is a cycle in Dag definition</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowDagDuplicatedIdException">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowDagDuplicatedIdException</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">dag_id</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a></span></em>, <em class="sig-param"><span class="n">incoming</span><span cl [...]
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when a Dag's ID is already used by another Dag</p>
+<dl class="py method">
+<dt id="airflow.exceptions.AirflowDagDuplicatedIdException.__str__">
+<code class="sig-name descname">__str__</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowDagDuplicatedIdException.__str__"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowDagDuplicatedIdException.__str__" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowClusterPolicyViolation">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowClusterPolicyViolation</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowClusterPolicyViolation"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowClusterPolicyViolation" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when there is a violation of a Cluster Policy in Dag definition</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.DagNotFound">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">DagNotFound</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#DagNotFound"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.DagNotFound" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowNotFoundException" title="airflow.exceptions.AirflowNotFoundException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowNotFoundException</span></code></a></p>
+<p>Raise when a DAG is not available in the system</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.DagCodeNotFound">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">DagCodeNotFound</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#DagCodeNotFound"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.DagCodeNotFound" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowNotFoundException" title="airflow.exceptions.AirflowNotFoundException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowNotFoundException</span></code></a></p>
+<p>Raise when a DAG code is not available in the system</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.DagRunNotFound">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">DagRunNotFound</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#DagRunNotFound"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.DagRunNotFound" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowNotFoundException" title="airflow.exceptions.AirflowNotFoundException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowNotFoundException</span></code></a></p>
+<p>Raise when a DAG Run is not available in the system</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.DagRunAlreadyExists">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">DagRunAlreadyExists</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#DagRunAlreadyExists"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.DagRunAlreadyExists" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowBadRequest" title="airflow.exceptions.AirflowBadRequest"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowBadRequest</span></code></a></p>
+<p>Raise when creating a DAG run for DAG which already has DAG run entry</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.DagFileExists">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">DagFileExists</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#DagFileExists"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.DagFileExists" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowBadRequest" title="airflow.exceptions.AirflowBadRequest"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowBadRequest</span></code></a></p>
+<p>Raise when a DAG ID is still in DagBag i.e., DAG file is in DAG folder</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.DuplicateTaskIdFound">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">DuplicateTaskIdFound</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#DuplicateTaskIdFound"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.DuplicateTaskIdFound" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when a Task with duplicate task_id is defined in the same DAG</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.SerializedDagNotFound">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">SerializedDagNotFound</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#SerializedDagNotFound"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.SerializedDagNotFound" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.DagNotFound" title="airflow.exceptions.DagNotFound"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.DagNotFound</span></code></a></p>
+<p>Raise when DAG is not found in the serialized_dags table in DB</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.SerializationError">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">SerializationError</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#SerializationError"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.SerializationError" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>A problem occurred when trying to serialize a DAG</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.TaskNotFound">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">TaskNotFound</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#TaskNotFound"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.TaskNotFound" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowNotFoundException" title="airflow.exceptions.AirflowNotFoundException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowNotFoundException</span></code></a></p>
+<p>Raise when a Task is not available in the system</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.TaskInstanceNotFound">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">TaskInstanceNotFound</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#TaskInstanceNotFound"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.TaskInstanceNotFound" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowNotFoundException" title="airflow.exceptions.AirflowNotFoundException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowNotFoundException</span></code></a></p>
+<p>Raise when a Task Instance is not available in the system</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.PoolNotFound">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">PoolNotFound</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#PoolNotFound"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.PoolNotFound" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowNotFoundException" title="airflow.exceptions.AirflowNotFoundException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowNotFoundException</span></code></a></p>
+<p>Raise when a Pool is not available in the system</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.NoAvailablePoolSlot">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">NoAvailablePoolSlot</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#NoAvailablePoolSlot"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.NoAvailablePoolSlot" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when there is not enough slots in pool</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.DagConcurrencyLimitReached">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">DagConcurrencyLimitReached</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#DagConcurrencyLimitReached"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.DagConcurrencyLimitReached" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when DAG concurrency limit is reached</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.TaskConcurrencyLimitReached">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">TaskConcurrencyLimitReached</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#TaskConcurrencyLimitReached"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.TaskConcurrencyLimitReached" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when task concurrency limit is reached</p>
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.BackfillUnfinished">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">BackfillUnfinished</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">message</span></em>, <em class="sig-param"><span class="n">ti_status</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/airflow/exceptions.html#BackfillUnfinished"><span class="viewcode-link">[source]</span></a><a cla [...]
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raises when not all tasks succeed in backfill.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>message</strong> -- The human-readable description of the exception</p></li>
+<li><p><strong>ti_status</strong> -- The information about all task statuses</p></li>
+</ul>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py class">
+<dt id="airflow.exceptions.FileSyntaxError">
+<em class="property">class </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">FileSyntaxError</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#FileSyntaxError"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.FileSyntaxError" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.NamedTuple" title="(in Python v3.9)"><code class="xref py py-class docutils literal notranslate"><span class="pre">typing.NamedTuple</span></code></a></p>
+<p>Information about a single error in a file.</p>
+<dl class="py attribute">
+<dt id="airflow.exceptions.FileSyntaxError.line_no">
+<code class="sig-name descname">line_no</code><em class="property"> :Optional[int]</em><a class="reference internal" href="../../../_modules/airflow/exceptions.html#FileSyntaxError.line_no"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.FileSyntaxError.line_no" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py attribute">
+<dt id="airflow.exceptions.FileSyntaxError.message">
+<code class="sig-name descname">message</code><em class="property"> :str</em><a class="reference internal" href="../../../_modules/airflow/exceptions.html#FileSyntaxError.message"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.FileSyntaxError.message" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.exceptions.FileSyntaxError.__str__">
+<code class="sig-name descname">__str__</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/airflow/exceptions.html#FileSyntaxError.__str__"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.FileSyntaxError.__str__" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.AirflowFileParseException">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">AirflowFileParseException</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">msg</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a></span></em>, <em class="sig-param"><span class="n">file_path</span><span class="p"> [...]
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raises when connection or variable file can not be parsed</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>msg</strong> -- The human-readable description of the exception</p></li>
+<li><p><strong>file_path</strong> -- A processed file that contains errors</p></li>
+<li><p><strong>parse_errors</strong> -- File syntax errors</p></li>
+</ul>
+</dd>
+</dl>
+<dl class="py method">
+<dt id="airflow.exceptions.AirflowFileParseException.__str__">
+<code class="sig-name descname">__str__</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../_modules/airflow/exceptions.html#AirflowFileParseException.__str__"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.AirflowFileParseException.__str__" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+</dd></dl>
+
+<dl class="py exception">
+<dt id="airflow.exceptions.ConnectionNotUnique">
+<em class="property">exception </em><code class="sig-prename descclassname">airflow.exceptions.</code><code class="sig-name descname">ConnectionNotUnique</code><a class="reference internal" href="../../../_modules/airflow/exceptions.html#ConnectionNotUnique"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.exceptions.ConnectionNotUnique" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="#airflow.exceptions.AirflowException" title="airflow.exceptions.AirflowException"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.exceptions.AirflowException</span></code></a></p>
+<p>Raise when multiple values are found for the same conn_id</p>
+</dd></dl>
+
</div>
</div>
<div class="pager" role="navigation" aria-label="related navigation">
- <a rel="prev" title="Building the image" href="build.html" >
+ <a rel="prev" title="airflow.models.xcom_arg" href="../models/xcom_arg/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="p">Previous</button>
</a>
- <a rel="next" title="Recipes" href="recipes.html" >
+ <a rel="next" title="airflow.secrets" href="../secrets/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="n">Next</button>
</a>
</div>
@@ -857,12 +1282,131 @@ To disable check, set <code class="docutils literal notranslate"><span class="pr
<nav class="wy-nav-side-toc">
<div class="wy-menu-vertical">
<ul>
-<li><a class="reference internal" href="#">Entrypoint</a><ul>
-<li><a class="reference internal" href="#allowing-arbitrary-user-to-run-the-container">Allowing arbitrary user to run the container</a></li>
-<li><a class="reference internal" href="#waits-for-airflow-db-connection">Waits for Airflow DB connection</a></li>
-<li><a class="reference internal" href="#upgrading-airflow-db">Upgrading Airflow DB</a></li>
-<li><a class="reference internal" href="#creating-admin-user">Creating admin user</a></li>
-<li><a class="reference internal" href="#waits-for-celery-broker-connection">Waits for celery broker connection</a></li>
+<li><a class="reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.exceptions</span></code></a><ul>
+<li><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowException">AirflowException</a><ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowException.status_code">status_code</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowBadRequest">AirflowBadRequest</a><ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowBadRequest.status_code">status_code</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowNotFoundException">AirflowNotFoundException</a><ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowNotFoundException.status_code">status_code</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowConfigException">AirflowConfigException</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowSensorTimeout">AirflowSensorTimeout</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowRescheduleException">AirflowRescheduleException</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowSmartSensorException">AirflowSmartSensorException</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.InvalidStatsNameException">InvalidStatsNameException</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowTaskTimeout">AirflowTaskTimeout</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowWebServerTimeout">AirflowWebServerTimeout</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowSkipException">AirflowSkipException</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowFailException">AirflowFailException</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowDagCycleException">AirflowDagCycleException</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowDagDuplicatedIdException">AirflowDagDuplicatedIdException</a><ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowDagDuplicatedIdException.__str__">__str__</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowClusterPolicyViolation">AirflowClusterPolicyViolation</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.DagNotFound">DagNotFound</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.DagCodeNotFound">DagCodeNotFound</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.DagRunNotFound">DagRunNotFound</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.DagRunAlreadyExists">DagRunAlreadyExists</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.DagFileExists">DagFileExists</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.DuplicateTaskIdFound">DuplicateTaskIdFound</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.SerializedDagNotFound">SerializedDagNotFound</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.SerializationError">SerializationError</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.TaskNotFound">TaskNotFound</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.TaskInstanceNotFound">TaskInstanceNotFound</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.PoolNotFound">PoolNotFound</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.NoAvailablePoolSlot">NoAvailablePoolSlot</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.DagConcurrencyLimitReached">DagConcurrencyLimitReached</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.TaskConcurrencyLimitReached">TaskConcurrencyLimitReached</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.BackfillUnfinished">BackfillUnfinished</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.FileSyntaxError">FileSyntaxError</a><ul>
+<li><a class="reference internal" href="#airflow.exceptions.FileSyntaxError.line_no">line_no</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.FileSyntaxError.message">message</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.FileSyntaxError.__str__">__str__</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowFileParseException">AirflowFileParseException</a><ul>
+<li><a class="reference internal" href="#airflow.exceptions.AirflowFileParseException.__str__">__str__</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.exceptions.ConnectionNotUnique">ConnectionNotUnique</a></li>
+</ul>
+</li>
</ul>
</li>
</ul>
@@ -887,7 +1431,7 @@ To disable check, set <code class="docutils literal notranslate"><span class="pr
<div class="base-layout--button">
- <a href="https://github.com/apache/airflow/edit/devel/docs/docker-stack/entrypoint.rst" rel="nofollow">
+ <a href="https://github.com/apache/airflow/edit/devel/docs/apache-airflow/_api/airflow/exceptions/index.rst" rel="nofollow">
<button class="btn-hollow btn-brown btn-with-icon with-box-shadow button-fixed">
<svg xmlns="http://www.w3.org/2000/svg" width="30.76" height="30">
@@ -1021,15 +1565,16 @@ To disable check, set <code class="docutils literal notranslate"><span class="pr
</span>
</div>
</footer>
- <script type="text/javascript" src="_static/_gen/js/docs.js"></script>
- <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../_static/_gen/js/docs.js"></script>
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
<script src="https://code.jquery.com/jquery-3.4.1.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script>
- <script src="_static/jquery.js"></script>
- <script src="_static/underscore.js"></script>
- <script src="_static/doctools.js"></script>
- <script src="_static/clipboard.min.js"></script>
- <script src="_static/copybutton.js"></script>
+ <script src="../../../_static/jquery.js"></script>
+ <script src="../../../_static/underscore.js"></script>
+ <script src="../../../_static/doctools.js"></script>
+ <script src="../../../_static/clipboard.min.js"></script>
+ <script src="../../../_static/copybutton.js"></script>
+ <script src="../../../_static/jira-links.js"></script>
</body>
</html>
\ No newline at end of file
diff --git a/docs-archive/docker-stack/recipes.html b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/base_executor/index.html
similarity index 58%
copy from docs-archive/docker-stack/recipes.html
copy to docs-archive/apache-airflow/2.1.0/_api/airflow/executors/base_executor/index.html
index 75019af..48635d8 100644
--- a/docs-archive/docker-stack/recipes.html
+++ b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/base_executor/index.html
@@ -9,15 +9,16 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Recipes — docker-stack Documentation</title>
- <link rel="stylesheet" href="_static/_gen/css/main.min.css" type="text/css" />
- <link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
- <link rel="stylesheet" type="text/css" href="_static/_gen/css/main-custom.min.css" />
- <link rel="shortcut icon" href="_static/pin_32.png"/>
- <link rel="index" title="Index" href="genindex.html" />
- <link rel="search" title="Search" href="search.html" />
- <link rel="next" title="Image build arguments reference" href="build-arg-ref.html" />
- <link rel="prev" title="Entrypoint" href="entrypoint.html" />
+ <title>airflow.executors.base_executor — Airflow Documentation</title>
+ <link rel="stylesheet" href="../../../../_static/_gen/css/main.min.css" type="text/css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/copybutton.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/_gen/css/main-custom.min.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/graphviz.css" />
+ <link rel="shortcut icon" href="../../../../_static/pin_32.png"/>
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+ <link rel="next" title="airflow.executors.celery_executor" href="../celery_executor/index.html" />
+ <link rel="prev" title="airflow.executors" href="../index.html" />
<script type="application/javascript">
var doNotTrack = false;
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
@@ -272,7 +273,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -283,7 +284,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -313,19 +314,163 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Recipes</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
-</ul>
-</li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.base_executor.PARALLELISM">PARALLELISM</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.base_executor.NOT_STARTED_MESSAGE">NOT_STARTED_MESSAGE</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.base_executor.CommandType">CommandType</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.base_executor.QueuedTaskInstanceType">QueuedTaskInstanceType</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.base_executor.EventBufferValueType">EventBufferValueType</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor">BaseExecutor</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.job_id">job_id</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.slots_available">slots_available</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.__repr__">__repr__</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.start">start</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.queue_command">queue_command</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.queue_task_instance">queue_task_instance</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.has_task">has_task</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.sync">sync</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.heartbeat">heartbeat</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.order_queued_tasks_by_priority">order_queued_tasks_by_priority</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.trigger_tasks">trigger_tasks</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.change_state">change_state</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.fail">fail</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.success">success</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.get_event_buffer">get_event_buffer</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.execute_async">execute_async</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.end">end</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.terminate">terminate</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.try_adopt_task_instances">try_adopt_task_instances</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.validate_command">validate_command</a></li>
+</ul>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.debug_dump">debug_dump</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -409,7 +554,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -420,7 +565,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -450,19 +595,163 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Recipes</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
-</ul>
-</li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.base_executor.PARALLELISM">PARALLELISM</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.base_executor.NOT_STARTED_MESSAGE">NOT_STARTED_MESSAGE</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.base_executor.CommandType">CommandType</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.base_executor.QueuedTaskInstanceType">QueuedTaskInstanceType</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.base_executor.EventBufferValueType">EventBufferValueType</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor">BaseExecutor</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.job_id">job_id</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.slots_available">slots_available</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.__repr__">__repr__</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.start">start</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.queue_command">queue_command</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.queue_task_instance">queue_task_instance</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.has_task">has_task</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.sync">sync</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.heartbeat">heartbeat</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.order_queued_tasks_by_priority">order_queued_tasks_by_priority</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.trigger_tasks">trigger_tasks</a></li>
+</ul>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.change_state">change_state</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.fail">fail</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.success">success</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.get_event_buffer">get_event_buffer</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.execute_async">execute_async</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.end">end</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.terminate">terminate</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.try_adopt_task_instances">try_adopt_task_instances</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.validate_command">validate_command</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.debug_dump">debug_dump</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -551,9 +840,13 @@
<ul class="breadcrumb">
- <li class="breadcrumb-item"><a href="index.html" class="icon icon-home"> Home</a></li>
+ <li class="breadcrumb-item"><a href="../../../../index.html" class="icon icon-home"> Home</a></li>
+
+ <li class="breadcrumb-item"><a href="../../../../python-api-ref.html">Python API Reference</a></li>
+
+ <li class="breadcrumb-item"><a href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a></li>
- <li class="breadcrumb-item"><a href="recipes.html"> Recipes</a></li>
+ <li class="breadcrumb-item"><a href="index.html"> <code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
</ul>
</div>
@@ -564,192 +857,260 @@
<div class="bodywrapper">
<div class="body" role="main">
- <blockquote>
-<div></div></blockquote>
-<div class="section" id="recipes">
-<h1>Recipes<a class="headerlink" href="#recipes" title="Permalink to this headline">¶</a></h1>
-<p>Users sometimes share interesting ways of using the Docker images. We encourage users to contribute these
-recipes to the documentation in case they prove useful to other members of the community by
-submitting a pull request. The sections below capture this knowledge.</p>
-<div class="section" id="google-cloud-sdk-installation">
-<h2>Google Cloud SDK installation<a class="headerlink" href="#google-cloud-sdk-installation" title="Permalink to this headline">¶</a></h2>
-<p>Some operators, such as <a class="reference external" href="/docs/apache-airflow-providers-google/stable/_api/airflow/providers/google/cloud/operators/kubernetes_engine/index.html#airflow.providers.google.cloud.operators.kubernetes_engine.GKEStartPodOperator" title="(in apache-airflow-providers-google vdevel)"><code class="xref py py-class docutils literal notranslate"><span class="pre">GKEStartPodOperator</span></code></a>,
-<a class="reference external" href="/docs/apache-airflow-providers-google/stable/_api/airflow/providers/google/cloud/operators/dataflow/index.html#airflow.providers.google.cloud.operators.dataflow.DataflowStartSqlJobOperator" title="(in apache-airflow-providers-google vdevel)"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataflowStartSqlJobOperator</span></code></a>, require
-the installation of <a class="reference external" href="https://cloud.google.com/sdk">Google Cloud SDK</a> (includes <code class="docutils literal notranslate"><span class="pre">gcloud</span></code>).
-You can also run these commands with BashOperator.</p>
-<p>Create a new Dockerfile like the one shown below.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-images-recipes/gcloud.Dockerfile</span></p>
-<div class="highlight-dockerfile notranslate"><div class="highlight"><pre><span></span><span class="c"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
-<span class="c"># contributor license agreements. See the NOTICE file distributed with</span>
-<span class="c"># this work for additional information regarding copyright ownership.</span>
-<span class="c"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
-<span class="c"># (the "License"); you may not use this file except in compliance with</span>
-<span class="c"># the License. You may obtain a copy of the License at</span>
-<span class="c">#</span>
-<span class="c"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c">#</span>
-<span class="c"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c"># See the License for the specific language governing permissions and</span>
-<span class="c"># limitations under the License.</span>
-<span class="k">ARG</span> BASE_AIRFLOW_IMAGE
-<span class="k">FROM</span> <span class="s">${BASE_AIRFLOW_IMAGE}</span>
-
-<span class="k">SHELL</span> <span class="p">[</span><span class="s2">"/bin/bash"</span><span class="p">,</span> <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"pipefail"</span><span class="p">,</span> <span class="s2">"-e"</span><span class="p">,</span> <span class="s2">"-u"</span><span class="p">,</span> <span class="s2">"-x"</span><span class="p">,</span> <span class="s2">"-c"</span><span clas [...]
-
-<span class="k">USER</span><span class="s"> 0</span>
-
-<span class="k">ARG</span> <span class="nv">CLOUD_SDK_VERSION</span><span class="o">=</span><span class="m">322</span>.0.0
-<span class="k">ENV</span> <span class="nv">GCLOUD_HOME</span><span class="o">=</span>/opt/google-cloud-sdk
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">/bin/:</span><span class="si">${</span><span class="nv">PATH</span><span class="si">}</span><span class="s2">"</span>
-
-<span class="k">RUN</span> <span class="nv">DOWNLOAD_URL</span><span class="o">=</span><span class="s2">"https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-</span><span class="si">${</span><span class="nv">CLOUD_SDK_VERSION</span><span class="si">}</span><span class="s2">-linux-x86_64.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="nv">TMP_DIR</span><span class="o">=</span><span class="s2">"</span><span class="k">$(</span>mktemp -d<span class="k">)</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> curl -fL <span class="s2">"</span><span class="si">${</span><span class="nv">DOWNLOAD_URL</span><span class="si">}</span><span class="s2">"</span> --output <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">/google-cloud-sdk.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir -p <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> tar xzf <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">/google-cloud-sdk.tar.gz"</span> -C <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">/install.sh"</span> <span class="se">\</span>
- --bash-completion<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --path-update<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --usage-reporting<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --additional-components alpha beta kubectl <span class="se">\</span>
- --quiet <span class="se">\</span>
- <span class="o">&&</span> rm -rf <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> gcloud --version
-
-<span class="k">USER</span><span class="s"> ${AIRFLOW_UID}</span>
-</pre></div>
-</div>
-</div>
-<p>Then build a new image.</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">BASE_AIRFLOW_IMAGE</span><span class="o">=</span><span class="s2">"apache/airflow:2.0.2"</span> <span class="se">\</span>
- -t my-airflow-image
-</pre></div>
-</div>
-</div>
-<div class="section" id="apache-hadoop-stack-installation">
-<h2>Apache Hadoop Stack installation<a class="headerlink" href="#apache-hadoop-stack-installation" title="Permalink to this headline">¶</a></h2>
-<p>Airflow is often used to run tasks on Hadoop cluster. It required Java Runtime Environment (JRE) to run.
-Below are the steps to take tools that are frequently used in Hadoop-world:</p>
-<ul class="simple">
-<li><p>Java Runtime Environment (JRE)</p></li>
-<li><p>Apache Hadoop</p></li>
-<li><p>Apache Hive</p></li>
-<li><p><a class="reference external" href="https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage">Cloud Storage connector for Apache Hadoop</a></p></li>
-</ul>
-<p>Create a new Dockerfile like the one shown below.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-images-recipes/hadoop.Dockerfile</span></p>
-<div class="highlight-dockerfile notranslate"><div class="highlight"><pre><span></span><span class="c"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
-<span class="c"># contributor license agreements. See the NOTICE file distributed with</span>
-<span class="c"># this work for additional information regarding copyright ownership.</span>
-<span class="c"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
-<span class="c"># (the "License"); you may not use this file except in compliance with</span>
-<span class="c"># the License. You may obtain a copy of the License at</span>
-<span class="c">#</span>
-<span class="c"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c">#</span>
-<span class="c"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c"># See the License for the specific language governing permissions and</span>
-<span class="c"># limitations under the License.</span>
-<span class="k">ARG</span> BASE_AIRFLOW_IMAGE
-
-<span class="k">FROM</span> <span class="s">${BASE_AIRFLOW_IMAGE}</span>
-
-<span class="k">SHELL</span> <span class="p">[</span><span class="s2">"/bin/bash"</span><span class="p">,</span> <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"pipefail"</span><span class="p">,</span> <span class="s2">"-e"</span><span class="p">,</span> <span class="s2">"-u"</span><span class="p">,</span> <span class="s2">"-x"</span><span class="p">,</span> <span class="s2">"-c"</span><span clas [...]
-
-<span class="k">USER</span><span class="s"> 0</span>
-
-<span class="c"># Install Java</span>
-<span class="k">RUN</span> mkdir -pv /usr/share/man/man1 <span class="se">\</span>
- <span class="o">&&</span> mkdir -pv /usr/share/man/man7 <span class="se">\</span>
- <span class="o">&&</span> curl -fsSL https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public <span class="p">|</span> apt-key add - <span class="se">\</span>
- <span class="o">&&</span> <span class="nb">echo</span> <span class="s1">'deb https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ buster main'</span> > <span class="se">\</span>
- /etc/apt/sources.list.d/adoptopenjdk.list <span class="se">\</span>
- <span class="o">&&</span> apt-get update <span class="se">\</span>
- <span class="o">&&</span> apt-get install --no-install-recommends -y <span class="se">\</span>
- adoptopenjdk-8-hotspot-jre <span class="se">\</span>
- <span class="o">&&</span> apt-get autoremove -yqq --purge <span class="se">\</span>
- <span class="o">&&</span> apt-get clean <span class="se">\</span>
- <span class="o">&&</span> rm -rf /var/lib/apt/lists/*
-<span class="k">ENV</span> <span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/adoptopenjdk-8-hotspot-jre-amd64
-
-<span class="k">RUN</span> mkdir -p /opt/spark/jars
-
-<span class="c"># Install Apache Hadoop</span>
-<span class="k">ARG</span> <span class="nv">HADOOP_VERSION</span><span class="o">=</span><span class="m">2</span>.10.1
-<span class="k">ENV</span> <span class="nv">HADOOP_HOME</span><span class="o">=</span>/opt/hadoop
-<span class="k">ENV</span> <span class="nv">HADOOP_CONF_DIR</span><span class="o">=</span>/etc/hadoop
-<span class="k">ENV</span> <span class="nv">MULTIHOMED_NETWORK</span><span class="o">=</span><span class="m">1</span>
-<span class="k">ENV</span> <span class="nv">USER</span><span class="o">=</span>root
-
-<span class="k">RUN</span> <span class="nv">HADOOP_URL</span><span class="o">=</span><span class="s2">"https://archive.apache.org/dist/hadoop/common/hadoop-</span><span class="nv">$HADOOP_VERSION</span><span class="s2">/hadoop-</span><span class="nv">$HADOOP_VERSION</span><span class="s2">.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> curl <span class="s1">'https://dist.apache.org/repos/dist/release/hadoop/common/KEYS'</span> <span class="p">|</span> gpg --import - <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HADOOP_URL</span><span class="s2">"</span> -o /tmp/hadoop.tar.gz <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HADOOP_URL</span><span class="s2">.asc"</span> -o /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> gpg --verify /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> tar -xvf /tmp/hadoop.tar.gz -C <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> rm /tmp/hadoop.tar.gz /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> ln -s <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">/etc/hadoop"</span> /etc/hadoop <span class="se">\</span>
- <span class="o">&&</span> mkdir <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">/logs"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir /hadoop-data
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="nv">$HADOOP_HOME</span><span class="s2">/bin/:</span><span class="nv">$PATH</span><span class="s2">"</span>
-
-<span class="c"># Install Apache Hive</span>
-<span class="k">ARG</span> <span class="nv">HIVE_VERSION</span><span class="o">=</span><span class="m">2</span>.3.7
-<span class="k">ENV</span> <span class="nv">HIVE_HOME</span><span class="o">=</span>/opt/hive
-<span class="k">ENV</span> <span class="nv">HIVE_CONF_DIR</span><span class="o">=</span>/etc/hive
-
-<span class="k">RUN</span> <span class="nv">HIVE_URL</span><span class="o">=</span><span class="s2">"https://archive.apache.org/dist/hive/hive-</span><span class="si">${</span><span class="nv">HIVE_VERSION</span><span class="si">}</span><span class="s2">/apache-hive-</span><span class="si">${</span><span class="nv">HIVE_VERSION</span><span class="si">}</span><span class="s2">-bin.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s1">'https://downloads.apache.org/hive/KEYS'</span> <span class="p">|</span> gpg --import - <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HIVE_URL</span><span class="s2">"</span> -o /tmp/hive.tar.gz <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HIVE_URL</span><span class="s2">.asc"</span> -o /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> gpg --verify /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> mkdir -p <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> tar -xf /tmp/hive.tar.gz -C <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> rm /tmp/hive.tar.gz /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> ln -s <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">/etc/hive"</span> <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_CONF_DIR</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">/logs"</span>
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="nv">$HIVE_HOME</span><span class="s2">/bin/:</span><span class="nv">$PATH</span><span class="s2">"</span>
-
-<span class="c"># Install GCS connector for Apache Hadoop</span>
-<span class="c"># See: https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage</span>
-<span class="k">ARG</span> <span class="nv">GCS_VARIANT</span><span class="o">=</span><span class="s2">"hadoop2"</span>
-<span class="k">ARG</span> <span class="nv">GCS_VERSION</span><span class="o">=</span><span class="s2">"2.1.5"</span>
-
-<span class="k">RUN</span> <span class="nv">GCS_JAR_PATH</span><span class="o">=</span><span class="s2">"/opt/spark/jars/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar"</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="nv">GCS_JAR_URL</span><span class="o">=</span><span class="s2">"https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar"</span> <span class="se">\</span>
- <span class="o">&&</span> curl <span class="s2">"</span><span class="si">${</span><span class="nv">GCS_JAR_URL</span><span class="si">}</span><span class="s2">"</span> -o <span class="s2">"</span><span class="si">${</span><span class="nv">GCS_JAR_PATH</span><span class="si">}</span><span class="s2">"</span>
-
-<span class="k">ENV</span> <span class="nv">HADOOP_CLASSPATH</span><span class="o">=</span><span class="s2">"/opt/spark/jars/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar:</span><span class="nv">$HADOOP_CLASSPATH</span><span class="s2">"</span>
-
-<span class="k">USER</span><span class="s"> ${AIRFLOW_UID}</span>
-</pre></div>
-</div>
-</div>
-<p>Then build a new image.</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">BASE_AIRFLOW_IMAGE</span><span class="o">=</span><span class="s2">"apache/airflow:2.0.2"</span> <span class="se">\</span>
- -t my-airflow-image
-</pre></div>
-</div>
+ <div class="section" id="module-airflow.executors.base_executor">
+<span id="airflow-executors-base-executor"></span><h1><a class="reference internal" href="#module-airflow.executors.base_executor" title="airflow.executors.base_executor"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a><a class="headerlink" href="#module-airflow.executors.base_executor" title="Permalink to this headline">¶</a></h1>
+<p><p>Base executor - this is the base class for all the implemented executors.</p>
+</p>
+<div class="section" id="module-contents">
+<h2>Module Contents<a class="headerlink" href="#module-contents" title="Permalink to this headline">¶</a></h2>
+<dl class="py data">
+<dt id="airflow.executors.base_executor.PARALLELISM">
+<code class="sig-prename descclassname">airflow.executors.base_executor.</code><code class="sig-name descname">PARALLELISM</code><em class="property"> :int</em><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#PARALLELISM"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.PARALLELISM" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.base_executor.NOT_STARTED_MESSAGE">
+<code class="sig-prename descclassname">airflow.executors.base_executor.</code><code class="sig-name descname">NOT_STARTED_MESSAGE</code><em class="property"> = The executor should be started first!</em><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#NOT_STARTED_MESSAGE"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.NOT_STARTED_MESSAGE" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.base_executor.CommandType">
+<code class="sig-prename descclassname">airflow.executors.base_executor.</code><code class="sig-name descname">CommandType</code><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#CommandType"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.CommandType" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.base_executor.QueuedTaskInstanceType">
+<code class="sig-prename descclassname">airflow.executors.base_executor.</code><code class="sig-name descname">QueuedTaskInstanceType</code><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#QueuedTaskInstanceType"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.QueuedTaskInstanceType" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.base_executor.EventBufferValueType">
+<code class="sig-prename descclassname">airflow.executors.base_executor.</code><code class="sig-name descname">EventBufferValueType</code><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#EventBufferValueType"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.EventBufferValueType" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py class">
+<dt id="airflow.executors.base_executor.BaseExecutor">
+<em class="property">class </em><code class="sig-prename descclassname">airflow.executors.base_executor.</code><code class="sig-name descname">BaseExecutor</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">parallelism</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.9)">int</a></span> <span class="o">=</span> <span class="default_value">PARALLELISM</span> [...]
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.utils.log.logging_mixin.LoggingMixin</span></code></p>
+<p>Class to derive in order to interface with executor-type systems
+like Celery, Kubernetes, Local, Sequential and the likes.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>parallelism</strong> -- how many jobs should run at one time. Set to
+<code class="docutils literal notranslate"><span class="pre">0</span></code> for infinity</p>
+</dd>
+</dl>
+<dl class="py attribute">
+<dt id="airflow.executors.base_executor.BaseExecutor.job_id">
+<code class="sig-name descname">job_id</code><em class="property"> :Optional[str]</em><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#BaseExecutor.job_id"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.BaseExecutor.job_id" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py attribute">
+<dt id="airflow.executors.base_executor.BaseExecutor.slots_available">
+<code class="sig-name descname">slots_available</code><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#BaseExecutor.slots_available"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.BaseExecutor.slots_available" title="Permalink to this definition">¶</a></dt>
+<dd><p>Number of new tasks this executor instance can accept</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.__repr__">
+<code class="sig-name descname">__repr__</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#BaseExecutor.__repr__"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.BaseExecutor.__repr__" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.start">
+<code class="sig-name descname">start</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#BaseExecutor.start"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.BaseExecutor.start" title="Permalink to this definition">¶</a></dt>
+<dd><p>Executors may need to get things started.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.queue_command">
+<code class="sig-name descname">queue_command</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">task_instance</span><span class="p">:</span> <span class="n">TaskInstance</span></em>, <em class="sig-param"><span class="n">command</span><span class="p">:</span> <span class="n"><a class="reference internal" href="#airflow.executors.base_executor.CommandType" title="airflow.executors.base_executor.CommandType"> [...]
+<dd><p>Queues command to task</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.queue_task_instance">
+<code class="sig-name descname">queue_task_instance</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">task_instance</span><span class="p">:</span> <span class="n">TaskInstance</span></em>, <em class="sig-param"><span class="n">mark_success</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.9)">bool</a [...]
+<dd><p>Queues task instance.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.has_task">
+<code class="sig-name descname">has_task</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">task_instance</span><span class="p">:</span> <span class="n">TaskInstance</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#BaseExecutor.has_task"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.e [...]
+<dd><p>Checks if a task is either queued or running in this executor.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>task_instance</strong> -- TaskInstance</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>True if the task is known to this executor</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.sync">
+<code class="sig-name descname">sync</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#BaseExecutor.sync"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.BaseExecutor.sync" title="Permalink to this definition">¶</a></dt>
+<dd><p>Sync will get called periodically by the heartbeat method.
+Executors should override this to perform gather statuses.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.heartbeat">
+<code class="sig-name descname">heartbeat</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#BaseExecutor.heartbeat"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.BaseExecutor.heartbeat" title="Permalink to this definition">¶</a></dt>
+<dd><p>Heartbeat sent to trigger new jobs.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.order_queued_tasks_by_priority">
+<code class="sig-name descname">order_queued_tasks_by_priority</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#BaseExecutor.order_queued_tasks_by_priority"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.BaseExecutor.order_queued_tasks_by_priority" title="Permal [...]
+<dd><p>Orders the queued tasks by priority.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>List of tuples from the queued_tasks according to the priority.</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.trigger_tasks">
+<code class="sig-name descname">trigger_tasks</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">open_slots</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.9)">int</a></span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.h [...]
+<dd><p>Triggers tasks</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>open_slots</strong> -- Number of open slots</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.change_state">
+<code class="sig-name descname">change_state</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">key</span><span class="p">:</span> <span class="n">TaskInstanceKey</span></em>, <em class="sig-param"><span class="n">state</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a></span></em>, <em class [...]
+<dd><p>Changes state of the task.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>info</strong> -- Executor information for the task instance</p></li>
+<li><p><strong>key</strong> -- Unique key for the task instance</p></li>
+<li><p><strong>state</strong> -- State to set for the task.</p></li>
+</ul>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.fail">
+<code class="sig-name descname">fail</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">key</span><span class="p">:</span> <span class="n">TaskInstanceKey</span></em>, <em class="sig-param"><span class="n">info</span><span class="o">=</span><span class="default_value">None</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html# [...]
+<dd><p>Set fail state for the event.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>info</strong> -- Executor information for the task instance</p></li>
+<li><p><strong>key</strong> -- Unique key for the task instance</p></li>
+</ul>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.success">
+<code class="sig-name descname">success</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">key</span><span class="p">:</span> <span class="n">TaskInstanceKey</span></em>, <em class="sig-param"><span class="n">info</span><span class="o">=</span><span class="default_value">None</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.ht [...]
+<dd><p>Set success state for the event.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>info</strong> -- Executor information for the task instance</p></li>
+<li><p><strong>key</strong> -- Unique key for the task instance</p></li>
+</ul>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.get_event_buffer">
+<code class="sig-name descname">get_event_buffer</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">dag_ids</span><span class="o">=</span><span class="default_value">None</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#BaseExecutor.get_event_buffer"><span class="viewcode-link">[source]</span></a><a class="headerlink" hre [...]
+<dd><p>Returns and flush the event buffer. In case dag_ids is specified
+it will only return and flush events for the given dag_ids. Otherwise
+it returns and flushes all events.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>dag_ids</strong> -- to dag_ids to return events for, if None returns all</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>a dict of events</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.execute_async">
+<code class="sig-name descname">execute_async</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">key</span><span class="p">:</span> <span class="n">TaskInstanceKey</span></em>, <em class="sig-param"><span class="n">command</span><span class="p">:</span> <span class="n"><a class="reference internal" href="#airflow.executors.base_executor.CommandType" title="airflow.executors.base_executor.CommandType">Command [...]
+<dd><p>This method will execute the command asynchronously.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>key</strong> -- Unique key for the task instance</p></li>
+<li><p><strong>command</strong> -- Command to run</p></li>
+<li><p><strong>queue</strong> -- name of the queue</p></li>
+<li><p><strong>executor_config</strong> -- Configuration passed to the executor.</p></li>
+</ul>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.end">
+<code class="sig-name descname">end</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#BaseExecutor.end"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.BaseExecutor.end" title="Permalink to this definition">¶</a></dt>
+<dd><p>This method is called when the caller is done submitting job and
+wants to wait synchronously for the job submitted previously to be
+all done.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.terminate">
+<code class="sig-name descname">terminate</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#BaseExecutor.terminate"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.BaseExecutor.terminate" title="Permalink to this definition">¶</a></dt>
+<dd><p>This method is called when the daemon receives a SIGTERM</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.try_adopt_task_instances">
+<code class="sig-name descname">try_adopt_task_instances</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">tis</span><span class="p">:</span> <span class="n">List<span class="p">[</span>TaskInstance<span class="p">]</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#BaseExecutor.try_adopt_task_instances"><span class [...]
+<dd><p>Try to adopt running task instances that have been abandoned by a SchedulerJob dying.</p>
+<p>Anything that is not adopted will be cleared by the scheduler (and then become eligible for
+re-scheduling)</p>
+<dl class="field-list simple">
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>any TaskInstances that were unable to be adopted</p>
+</dd>
+<dt class="field-even">Return type</dt>
+<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.9)">list</a>[<a class="reference internal" href="../../models/index.html#airflow.models.TaskInstance" title="airflow.models.TaskInstance">airflow.models.TaskInstance</a>]</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.validate_command">
+<em class="property">static </em><code class="sig-name descname">validate_command</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">command</span><span class="p">:</span> <span class="n">List<span class="p">[</span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a><span class="p">]</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airfl [...]
+<dd><p>Check if the command to execute is airflow command</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.base_executor.BaseExecutor.debug_dump">
+<code class="sig-name descname">debug_dump</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/base_executor.html#BaseExecutor.debug_dump"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.base_executor.BaseExecutor.debug_dump" title="Permalink to this definition">¶</a></dt>
+<dd><p>Called in response to SIGUSR2 by the scheduler</p>
+</dd></dl>
+
+</dd></dl>
+
</div>
</div>
<div class="pager" role="navigation" aria-label="related navigation">
- <a rel="prev" title="Entrypoint" href="entrypoint.html" >
+ <a rel="prev" title="airflow.executors" href="../index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="p">Previous</button>
</a>
- <a rel="next" title="Image build arguments reference" href="build-arg-ref.html" >
+ <a rel="next" title="airflow.executors.celery_executor" href="../celery_executor/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="n">Next</button>
</a>
</div>
@@ -819,9 +1180,89 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
<nav class="wy-nav-side-toc">
<div class="wy-menu-vertical">
<ul>
-<li><a class="reference internal" href="#">Recipes</a><ul>
-<li><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
+<li><a class="reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a><ul>
+<li><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.PARALLELISM">PARALLELISM</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.NOT_STARTED_MESSAGE">NOT_STARTED_MESSAGE</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.CommandType">CommandType</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.QueuedTaskInstanceType">QueuedTaskInstanceType</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.EventBufferValueType">EventBufferValueType</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor">BaseExecutor</a><ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.job_id">job_id</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.slots_available">slots_available</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.__repr__">__repr__</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.start">start</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.queue_command">queue_command</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.queue_task_instance">queue_task_instance</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.has_task">has_task</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.sync">sync</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.heartbeat">heartbeat</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.order_queued_tasks_by_priority">order_queued_tasks_by_priority</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.trigger_tasks">trigger_tasks</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.change_state">change_state</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.fail">fail</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.success">success</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.get_event_buffer">get_event_buffer</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.execute_async">execute_async</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.end">end</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.terminate">terminate</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.try_adopt_task_instances">try_adopt_task_instances</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.validate_command">validate_command</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.base_executor.BaseExecutor.debug_dump">debug_dump</a></li>
+</ul>
+</li>
+</ul>
+</li>
</ul>
</li>
</ul>
@@ -846,7 +1287,7 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
<div class="base-layout--button">
- <a href="https://github.com/apache/airflow/edit/devel/docs/docker-stack/recipes.rst" rel="nofollow">
+ <a href="https://github.com/apache/airflow/edit/devel/docs/apache-airflow/_api/airflow/executors/base_executor/index.rst" rel="nofollow">
<button class="btn-hollow btn-brown btn-with-icon with-box-shadow button-fixed">
<svg xmlns="http://www.w3.org/2000/svg" width="30.76" height="30">
@@ -980,15 +1421,16 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
</span>
</div>
</footer>
- <script type="text/javascript" src="_static/_gen/js/docs.js"></script>
- <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/_gen/js/docs.js"></script>
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
<script src="https://code.jquery.com/jquery-3.4.1.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script>
- <script src="_static/jquery.js"></script>
- <script src="_static/underscore.js"></script>
- <script src="_static/doctools.js"></script>
- <script src="_static/clipboard.min.js"></script>
- <script src="_static/copybutton.js"></script>
+ <script src="../../../../_static/jquery.js"></script>
+ <script src="../../../../_static/underscore.js"></script>
+ <script src="../../../../_static/doctools.js"></script>
+ <script src="../../../../_static/clipboard.min.js"></script>
+ <script src="../../../../_static/copybutton.js"></script>
+ <script src="../../../../_static/jira-links.js"></script>
</body>
</html>
\ No newline at end of file
diff --git a/docs-archive/docker-stack/build.html b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/celery_executor/index.html
similarity index 53%
copy from docs-archive/docker-stack/build.html
copy to docs-archive/apache-airflow/2.1.0/_api/airflow/executors/celery_executor/index.html
index ef84de3..11c89e5 100644
--- a/docs-archive/docker-stack/build.html
+++ b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/celery_executor/index.html
@@ -9,15 +9,16 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Building the image — docker-stack Documentation</title>
- <link rel="stylesheet" href="_static/_gen/css/main.min.css" type="text/css" />
- <link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
- <link rel="stylesheet" type="text/css" href="_static/_gen/css/main-custom.min.css" />
- <link rel="shortcut icon" href="_static/pin_32.png"/>
- <link rel="index" title="Index" href="genindex.html" />
- <link rel="search" title="Search" href="search.html" />
- <link rel="next" title="Entrypoint" href="entrypoint.html" />
- <link rel="prev" title="Docker Image for Apache Airflow" href="index.html" />
+ <title>airflow.executors.celery_executor — Airflow Documentation</title>
+ <link rel="stylesheet" href="../../../../_static/_gen/css/main.min.css" type="text/css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/copybutton.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/_gen/css/main-custom.min.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/graphviz.css" />
+ <link rel="shortcut icon" href="../../../../_static/pin_32.png"/>
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+ <link rel="next" title="airflow.executors.celery_kubernetes_executor" href="../celery_kubernetes_executor/index.html" />
+ <link rel="prev" title="airflow.executors.base_executor" href="../base_executor/index.html" />
<script type="application/javascript">
var doNotTrack = false;
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
@@ -272,7 +273,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -283,7 +284,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -313,42 +314,193 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Building the image</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#extending-vs-customizing-the-image">Extending vs. customizing the image</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#extending-the-image">Extending the image</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#examples-of-image-extending">Examples of image extending</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="#an-apt-package-example">An <code class="docutils literal notranslate"><span class="pre">apt</span></code> package example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#a-pypi-package-example">A <code class="docutils literal notranslate"><span class="pre">PyPI</span></code> package example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#a-umask-requiring-example">A <code class="docutils literal notranslate"><span class="pre">umask</span></code> requiring example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#a-build-essential-requiring-package-example">A <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> requiring package example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#dag-embedding-example">DAG embedding example</a></li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
+</ul>
+<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6"><a class="reference internal" href="../base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.log">log</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.CELERY_FETCH_ERR_MSG_HEADER">CELERY_FETCH_ERR_MSG_HEADER</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.CELERY_SEND_ERR_MSG_HEADER">CELERY_SEND_ERR_MSG_HEADER</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.OPERATION_TIMEOUT">OPERATION_TIMEOUT</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.celery_configuration">celery_configuration</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.app">app</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.execute_command">execute_command</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor._execute_in_fork">_execute_in_fork</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor._execute_in_subprocess">_execute_in_subprocess</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.ExceptionWithTraceback">ExceptionWithTraceback</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.TaskInstanceInCelery">TaskInstanceInCelery</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.send_task_to_executor">send_task_to_executor</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.on_celery_import_modules">on_celery_import_modules</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor">CeleryExecutor</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.start">start</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor._num_tasks_per_send_process">_num_tasks_per_send_process</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.trigger_tasks">trigger_tasks</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor._process_tasks">_process_tasks</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor._send_tasks_to_celery">_send_tasks_to_celery</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.sync">sync</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor._check_for_stalled_adopted_tasks">_check_for_stalled_adopted_tasks</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.debug_dump">debug_dump</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.update_all_task_states">update_all_task_states</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.change_state">change_state</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.update_task_state">update_task_state</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.end">end</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.execute_async">execute_async</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.terminate">terminate</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.try_adopt_task_instances">try_adopt_task_instances</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="#customizing-the-image">Customizing the image</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#examples-of-image-customizing">Examples of image customizing</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="#building-from-pypi-packages">Building from PyPI packages</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#building-optimized-images">Building optimized images</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#building-from-github">Building from GitHub</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#using-custom-installation-sources">Using custom installation sources</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#build-images-in-security-restricted-environments">Build images in security restricted environments</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#modifying-the-dockerfile">Modifying the Dockerfile</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.fetch_celery_task_state">fetch_celery_task_state</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher">BulkStateFetcher</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._tasks_list_to_task_ids">_tasks_list_to_task_ids</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher.get_many">get_many</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._get_many_from_kv_backend">_get_many_from_kv_backend</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._get_many_from_db_backend">_get_many_from_db_backend</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._prepare_state_and_info_by_task_dict">_prepare_state_and_info_by_task_dict</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._get_many_using_multiprocessing">_get_many_using_multiprocessing</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="#more-details">More details</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="#build-args-reference">Build Args reference</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#the-architecture-of-the-images">The architecture of the images</a></li>
</ul>
</li>
</ul>
</li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1"><a class="reference internal" href="recipes.html">Recipes</a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
</ul>
-<p class="caption"><span class="caption-text">References</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -432,7 +584,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -443,7 +595,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -473,42 +625,193 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Building the image</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#extending-vs-customizing-the-image">Extending vs. customizing the image</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#extending-the-image">Extending the image</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#examples-of-image-extending">Examples of image extending</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="#an-apt-package-example">An <code class="docutils literal notranslate"><span class="pre">apt</span></code> package example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#a-pypi-package-example">A <code class="docutils literal notranslate"><span class="pre">PyPI</span></code> package example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#a-umask-requiring-example">A <code class="docutils literal notranslate"><span class="pre">umask</span></code> requiring example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#a-build-essential-requiring-package-example">A <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> requiring package example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#dag-embedding-example">DAG embedding example</a></li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
+</ul>
+<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6"><a class="reference internal" href="../base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.log">log</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.CELERY_FETCH_ERR_MSG_HEADER">CELERY_FETCH_ERR_MSG_HEADER</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.CELERY_SEND_ERR_MSG_HEADER">CELERY_SEND_ERR_MSG_HEADER</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.OPERATION_TIMEOUT">OPERATION_TIMEOUT</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.celery_configuration">celery_configuration</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.app">app</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.execute_command">execute_command</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor._execute_in_fork">_execute_in_fork</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor._execute_in_subprocess">_execute_in_subprocess</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.ExceptionWithTraceback">ExceptionWithTraceback</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.TaskInstanceInCelery">TaskInstanceInCelery</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.send_task_to_executor">send_task_to_executor</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.on_celery_import_modules">on_celery_import_modules</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor">CeleryExecutor</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.start">start</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor._num_tasks_per_send_process">_num_tasks_per_send_process</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.trigger_tasks">trigger_tasks</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor._process_tasks">_process_tasks</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor._send_tasks_to_celery">_send_tasks_to_celery</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.sync">sync</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor._check_for_stalled_adopted_tasks">_check_for_stalled_adopted_tasks</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.debug_dump">debug_dump</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.update_all_task_states">update_all_task_states</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.change_state">change_state</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.update_task_state">update_task_state</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.end">end</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.execute_async">execute_async</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.terminate">terminate</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.try_adopt_task_instances">try_adopt_task_instances</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.fetch_celery_task_state">fetch_celery_task_state</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher">BulkStateFetcher</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._tasks_list_to_task_ids">_tasks_list_to_task_ids</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher.get_many">get_many</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._get_many_from_kv_backend">_get_many_from_kv_backend</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._get_many_from_db_backend">_get_many_from_db_backend</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._prepare_state_and_info_by_task_dict">_prepare_state_and_info_by_task_dict</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._get_many_using_multiprocessing">_get_many_using_multiprocessing</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="#customizing-the-image">Customizing the image</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#examples-of-image-customizing">Examples of image customizing</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="#building-from-pypi-packages">Building from PyPI packages</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#building-optimized-images">Building optimized images</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#building-from-github">Building from GitHub</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#using-custom-installation-sources">Using custom installation sources</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#build-images-in-security-restricted-environments">Build images in security restricted environments</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#modifying-the-dockerfile">Modifying the Dockerfile</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="#more-details">More details</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="#build-args-reference">Build Args reference</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#the-architecture-of-the-images">The architecture of the images</a></li>
</ul>
</li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
</ul>
</li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1"><a class="reference internal" href="recipes.html">Recipes</a></li>
</ul>
-<p class="caption"><span class="caption-text">References</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -597,9 +900,13 @@
<ul class="breadcrumb">
- <li class="breadcrumb-item"><a href="index.html" class="icon icon-home"> Home</a></li>
+ <li class="breadcrumb-item"><a href="../../../../index.html" class="icon icon-home"> Home</a></li>
+
+ <li class="breadcrumb-item"><a href="../../../../python-api-ref.html">Python API Reference</a></li>
+
+ <li class="breadcrumb-item"><a href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a></li>
- <li class="breadcrumb-item"><a href="build.html"> Building the image</a></li>
+ <li class="breadcrumb-item"><a href="index.html"> <code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
</ul>
</div>
@@ -610,627 +917,288 @@
<div class="bodywrapper">
<div class="body" role="main">
- <blockquote>
-<div></div></blockquote>
-<div class="section" id="building-the-image">
-<h1>Building the image<a class="headerlink" href="#building-the-image" title="Permalink to this headline">¶</a></h1>
-<p>Before you dive-deeply in the way how the Airflow Image is build, named and why we are doing it the
-way we do, you might want to know very quickly how you can extend or customize the existing image
-for Apache Airflow. This chapter gives you a short answer to those questions.</p>
-<div class="section" id="extending-vs-customizing-the-image">
-<h2>Extending vs. customizing the image<a class="headerlink" href="#extending-vs-customizing-the-image" title="Permalink to this headline">¶</a></h2>
-<p>Here is the comparison of the two types of building images. Here is your guide if you want to choose
-how you want to build your image.</p>
-<table class="docutils align-default">
-<colgroup>
-<col style="width: 68%" />
-<col style="width: 14%" />
-<col style="width: 17%" />
-</colgroup>
-<thead>
-<tr class="row-odd"><th class="head"></th>
-<th class="head"><p>Extending</p></th>
-<th class="head"><p>Customizing</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p>Can be built without airflow sources</p></td>
-<td><p>Yes</p></td>
-<td><p>No</p></td>
-</tr>
-<tr class="row-odd"><td><p>Uses familiar 'FROM ' pattern of image building</p></td>
-<td><p>Yes</p></td>
-<td><p>No</p></td>
-</tr>
-<tr class="row-even"><td><p>Requires only basic knowledge about images</p></td>
-<td><p>Yes</p></td>
-<td><p>No</p></td>
-</tr>
-<tr class="row-odd"><td><p>Builds quickly</p></td>
-<td><p>Yes</p></td>
-<td><p>No</p></td>
-</tr>
-<tr class="row-even"><td><p>Produces image heavily optimized for size</p></td>
-<td><p>No</p></td>
-<td><p>Yes</p></td>
-</tr>
-<tr class="row-odd"><td><p>Can build from custom airflow sources (forks)</p></td>
-<td><p>No</p></td>
-<td><p>Yes</p></td>
-</tr>
-<tr class="row-even"><td><p>Can build on air-gaped system</p></td>
-<td><p>No</p></td>
-<td><p>Yes</p></td>
-</tr>
-</tbody>
-</table>
-<p>TL;DR; If you have a need to build custom image, it is easier to start with "Extending" however if your
-dependencies require compilation step or when your require to build the image from security vetted
-packages, switching to "Customizing" the image provides much more optimized images. In the example further
-where we compare equivalent "Extending" and "Customizing" the image, similar images build by
-Extending vs. Customization had shown 1.1GB vs 874MB image sizes respectively - with 20% improvement in
-size of the Customized image.</p>
-<div class="admonition note">
-<p class="admonition-title">Note</p>
-<p>You can also combine both - customizing & extending the image in one. You can build your
-optimized base image first using <code class="docutils literal notranslate"><span class="pre">customization</span></code> method (for example by your admin team) with all
-the heavy compilation required dependencies and you can publish it in your registry and let others
-<code class="docutils literal notranslate"><span class="pre">extend</span></code> your image using <code class="docutils literal notranslate"><span class="pre">FROM</span></code> and add their own lightweight dependencies. This reflects well
-the split where typically "Casual" users will Extend the image and "Power-users" will customize it.</p>
-</div>
-<p>Airflow Summit 2020's <a class="reference external" href="https://youtu.be/wDr3Y7q2XoI">Production Docker Image</a> talk provides more
-details about the context, architecture and customization/extension methods for the Production Image.</p>
-</div>
-<div class="section" id="extending-the-image">
-<h2>Extending the image<a class="headerlink" href="#extending-the-image" title="Permalink to this headline">¶</a></h2>
-<p>Extending the image is easiest if you just need to add some dependencies that do not require
-compiling. The compilation framework of Linux (so called <code class="docutils literal notranslate"><span class="pre">build-essential</span></code>) is pretty big, and
-for the production images, size is really important factor to optimize for, so our Production Image
-does not contain <code class="docutils literal notranslate"><span class="pre">build-essential</span></code>. If you need compiler like gcc or g++ or make/cmake etc. - those
-are not found in the image and it is recommended that you follow the "customize" route instead.</p>
-<p>How to extend the image - it is something you are most likely familiar with - simply
-build a new image using Dockerfile's <code class="docutils literal notranslate"><span class="pre">FROM</span></code> directive and add whatever you need. Then you can add your
-Debian dependencies with <code class="docutils literal notranslate"><span class="pre">apt</span></code> or PyPI dependencies with <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> or any other stuff you need.</p>
-<p>You should be aware, about a few things:</p>
-<ul class="simple">
-<li><p>The production image of airflow uses "airflow" user, so if you want to add some of the tools
-as <code class="docutils literal notranslate"><span class="pre">root</span></code> user, you need to switch to it with <code class="docutils literal notranslate"><span class="pre">USER</span></code> directive of the Dockerfile and switch back to
-<code class="docutils literal notranslate"><span class="pre">airflow</span></code> user when you are done. Also you should remember about following the
-<a class="reference external" href="https://docs.docker.com/develop/develop-images/dockerfile_best-practices/">best practises of Dockerfiles</a>
-to make sure your image is lean and small.</p></li>
-<li><p>The PyPI dependencies in Apache Airflow are installed in the user library, of the "airflow" user, so
-PIP packages are installed to <code class="docutils literal notranslate"><span class="pre">~/.local</span></code> folder as if the <code class="docutils literal notranslate"><span class="pre">--user</span></code> flag was specified when running PIP.
-Note also that using <code class="docutils literal notranslate"><span class="pre">--no-cache-dir</span></code> is a good idea that can help to make your image smaller.</p></li>
-</ul>
-<div class="admonition note">
-<p class="admonition-title">Note</p>
-<p>Only as of <code class="docutils literal notranslate"><span class="pre">2.0.1</span></code> image the <code class="docutils literal notranslate"><span class="pre">--user</span></code> flag is turned on by default by setting <code class="docutils literal notranslate"><span class="pre">PIP_USER</span></code> environment
-variable to <code class="docutils literal notranslate"><span class="pre">true</span></code>. This can be disabled by un-setting the variable or by setting it to <code class="docutils literal notranslate"><span class="pre">false</span></code>. In the
-2.0.0 image you had to add the <code class="docutils literal notranslate"><span class="pre">--user</span></code> flag as <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--user</span></code> command.</p>
-</div>
-<ul class="simple">
-<li><p>If your apt, or PyPI dependencies require some of the <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> or other packages that need
-to compile your python dependencies, then your best choice is to follow the "Customize the image" route,
-because you can build a highly-optimized (for size) image this way. However it requires to checkout sources
-of Apache Airflow, so you might still want to choose to add <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> to your image,
-even if your image will be significantly bigger.</p></li>
-<li><p>You can also embed your dags in the image by simply adding them with COPY directive of Airflow.
-The DAGs in production image are in <code class="docutils literal notranslate"><span class="pre">/opt/airflow/dags</span></code> folder.</p></li>
-<li><p>You can build your image without any need for Airflow sources. It is enough that you place the
-<code class="docutils literal notranslate"><span class="pre">Dockerfile</span></code> and any files that are referred to (such as Dag files) in a separate directory and run
-a command <code class="docutils literal notranslate"><span class="pre">docker</span> <span class="pre">build</span> <span class="pre">.</span> <span class="pre">--tag</span> <span class="pre">my-image:my-tag</span></code> (where <code class="docutils literal notranslate"><span class="pre">my-image</span></code> is the name you want to name it
-and <code class="docutils literal notranslate"><span class="pre">my-tag</span></code> is the tag you want to tag the image with.</p></li>
-<li><p>If your way of extending image requires to create writable directories, you MUST remember about adding
-<code class="docutils literal notranslate"><span class="pre">umask</span> <span class="pre">0002</span></code> step in your RUN command. This is necessary in order to accommodate our approach for
-running the image with an arbitrary user. Such user will always run with <code class="docutils literal notranslate"><span class="pre">GID=0</span></code> -
-the entrypoint will prevent non-root GIDs. You can read more about it in
-<a class="reference internal" href="entrypoint.html#arbitrary-docker-user"><span class="std std-ref">arbitrary docker user</span></a> documentation for the entrypoint. The
-<code class="docutils literal notranslate"><span class="pre">umask</span> <span class="pre">0002</span></code> is set as default when you enter the image, so any directories you create by default
-in runtime, will have <code class="docutils literal notranslate"><span class="pre">GID=0</span></code> and will be group-writable.</p></li>
-</ul>
-<div class="admonition note">
-<p class="admonition-title">Note</p>
-<p>Only as of <code class="docutils literal notranslate"><span class="pre">2.0.2</span></code> the default group of <code class="docutils literal notranslate"><span class="pre">airflow</span></code> user is <code class="docutils literal notranslate"><span class="pre">root</span></code>. Previously it was <code class="docutils literal notranslate"><span class="pre">airflow</span></code>,
-so if you are building your images based on an earlier image, you need to manually change the default
-group for airflow user:</p>
-</div>
-<div class="highlight-docker notranslate"><div class="highlight"><pre><span></span><span class="k">RUN</span> usermod -g <span class="m">0</span> airflow
-</pre></div>
-</div>
-</div>
-<div class="section" id="examples-of-image-extending">
-<h2>Examples of image extending<a class="headerlink" href="#examples-of-image-extending" title="Permalink to this headline">¶</a></h2>
-<div class="section" id="an-apt-package-example">
-<h3>An <code class="docutils literal notranslate"><span class="pre">apt</span></code> package example<a class="headerlink" href="#an-apt-package-example" title="Permalink to this headline">¶</a></h3>
-<p>The following example adds <code class="docutils literal notranslate"><span class="pre">vim</span></code> to the airflow image.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/extending/add-apt-packages/Dockerfile</span></p>
-<div class="highlight-Dockerfile notranslate"><div class="highlight"><pre><span></span><span class="k">FROM</span> <span class="s">apache/airflow:2.0.2</span>
-<span class="k">USER</span><span class="s"> root</span>
-<span class="k">RUN</span> apt-get update <span class="se">\</span>
- <span class="o">&&</span> apt-get install -y --no-install-recommends <span class="se">\</span>
- vim <span class="se">\</span>
- <span class="o">&&</span> apt-get autoremove -yqq --purge <span class="se">\</span>
- <span class="o">&&</span> apt-get clean <span class="se">\</span>
- <span class="o">&&</span> rm -rf /var/lib/apt/lists/*
-<span class="k">USER</span><span class="s"> airflow</span>
-</pre></div>
-</div>
-</div>
-</div>
-<div class="section" id="a-pypi-package-example">
-<h3>A <code class="docutils literal notranslate"><span class="pre">PyPI</span></code> package example<a class="headerlink" href="#a-pypi-package-example" title="Permalink to this headline">¶</a></h3>
-<p>The following example adds <code class="docutils literal notranslate"><span class="pre">lxml</span></code> python package from PyPI to the image.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/extending/add-pypi-packages/Dockerfile</span></p>
-<div class="highlight-Dockerfile notranslate"><div class="highlight"><pre><span></span><span class="k">FROM</span> <span class="s">apache/airflow:2.0.2</span>
-<span class="k">RUN</span> pip install --no-cache-dir lxml
-</pre></div>
-</div>
-</div>
-</div>
-<div class="section" id="a-umask-requiring-example">
-<h3>A <code class="docutils literal notranslate"><span class="pre">umask</span></code> requiring example<a class="headerlink" href="#a-umask-requiring-example" title="Permalink to this headline">¶</a></h3>
-<p>The following example adds a new directory that is supposed to be writable for any arbitrary user
-running the container.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/extending/writable-directory/Dockerfile</span></p>
-<div class="highlight-Dockerfile notranslate"><div class="highlight"><pre><span></span><span class="k">FROM</span> <span class="s">apache/airflow:2.0.2</span>
-<span class="k">RUN</span> <span class="nb">umask</span> <span class="m">0002</span><span class="p">;</span> <span class="se">\</span>
- mkdir -p ~/writeable-directory
-</pre></div>
-</div>
-</div>
-</div>
-<div class="section" id="a-build-essential-requiring-package-example">
-<h3>A <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> requiring package example<a class="headerlink" href="#a-build-essential-requiring-package-example" title="Permalink to this headline">¶</a></h3>
-<p>The following example adds <code class="docutils literal notranslate"><span class="pre">mpi4py</span></code> package which requires both <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> and <code class="docutils literal notranslate"><span class="pre">mpi</span> <span class="pre">compiler</span></code>.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/extending/add-build-essential-extend/Dockerfile</span></p>
-<div class="highlight-Dockerfile notranslate"><div class="highlight"><pre><span></span><span class="k">FROM</span> <span class="s">apache/airflow:2.0.2</span>
-<span class="k">USER</span><span class="s"> root</span>
-<span class="k">RUN</span> apt-get update <span class="se">\</span>
- <span class="o">&&</span> apt-get install -y --no-install-recommends <span class="se">\</span>
- build-essential libopenmpi-dev <span class="se">\</span>
- <span class="o">&&</span> apt-get autoremove -yqq --purge <span class="se">\</span>
- <span class="o">&&</span> apt-get clean <span class="se">\</span>
- <span class="o">&&</span> rm -rf /var/lib/apt/lists/*
-<span class="k">USER</span><span class="s"> airflow</span>
-<span class="k">RUN</span> pip install --no-cache-dir mpi4py
-</pre></div>
-</div>
-</div>
-<p>The size of this image is ~ 1.1 GB when build. As you will see further, you can achieve 20% reduction in
-size of the image in case you use "Customizing" rather than "Extending" the image.</p>
-</div>
-<div class="section" id="dag-embedding-example">
-<h3>DAG embedding example<a class="headerlink" href="#dag-embedding-example" title="Permalink to this headline">¶</a></h3>
-<p>The following example adds <code class="docutils literal notranslate"><span class="pre">test_dag.py</span></code> to your image in the <code class="docutils literal notranslate"><span class="pre">/opt/airflow/dags</span></code> folder.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/extending/embedding-dags/Dockerfile</span></p>
-<div class="highlight-Dockerfile notranslate"><div class="highlight"><pre><span></span><span class="k">FROM</span> <span class="s">apache/airflow:2.0.2</span>
-
-<span class="k">COPY</span> --chown<span class="o">=</span>airflow:root test_dag.py /opt/airflow/dags
-</pre></div>
-</div>
-</div>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/extending/embedding-dags/test_dag.py</span></p>
-<div class="highlight-Python notranslate"><div class="highlight"><pre><span></span><span class="sd">"""This dag only runs some simple tasks to test Airflow's task execution."""</span>
-<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
-
-<span class="kn">from</span> <span class="nn">airflow.models.dag</span> <span class="kn">import</span> <span class="n">DAG</span>
-<span class="kn">from</span> <span class="nn">airflow.operators.dummy</span> <span class="kn">import</span> <span class="n">DummyOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.dates</span> <span class="kn">import</span> <span class="n">days_ago</span>
-
-<span class="n">now</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span>
-<span class="n">now_to_the_hour</span> <span class="o">=</span> <span class="p">(</span><span class="n">now</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</s [...]
-<span class="n">START_DATE</span> <span class="o">=</span> <span class="n">now_to_the_hour</span>
-<span class="n">DAG_NAME</span> <span class="o">=</span> <span class="s1">'test_dag_v1'</span>
-
-<span class="n">default_args</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'owner'</span><span class="p">:</span> <span class="s1">'airflow'</span><span class="p">,</span> <span class="s1">'depends_on_past'</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span> <span class="s1">'start_date'</span><span class="p">:</span> <span class="n">days_ago</span><span class="p">(</span><span class="mi">2</span>< [...]
-<span class="n">dag</span> <span class="o">=</span> <span class="n">DAG</span><span class="p">(</span><span class="n">DAG_NAME</span><span class="p">,</span> <span class="n">schedule_interval</span><span class="o">=</span><span class="s1">'*/10 * * * *'</span><span class="p">,</span> <span class="n">default_args</span><span class="o">=</span><span class="n">default_args</span><span class="p">)</span>
-
-<span class="n">run_this_1</span> <span class="o">=</span> <span class="n">DummyOperator</span><span class="p">(</span><span class="n">task_id</span><span class="o">=</span><span class="s1">'run_this_1'</span><span class="p">,</span> <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
-<span class="n">run_this_2</span> <span class="o">=</span> <span class="n">DummyOperator</span><span class="p">(</span><span class="n">task_id</span><span class="o">=</span><span class="s1">'run_this_2'</span><span class="p">,</span> <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
-<span class="n">run_this_2</span><span class="o">.</span><span class="n">set_upstream</span><span class="p">(</span><span class="n">run_this_1</span><span class="p">)</span>
-<span class="n">run_this_3</span> <span class="o">=</span> <span class="n">DummyOperator</span><span class="p">(</span><span class="n">task_id</span><span class="o">=</span><span class="s1">'run_this_3'</span><span class="p">,</span> <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
-<span class="n">run_this_3</span><span class="o">.</span><span class="n">set_upstream</span><span class="p">(</span><span class="n">run_this_2</span><span class="p">)</span>
-</pre></div>
-</div>
-</div>
-</div>
-</div>
-<div class="section" id="customizing-the-image">
-<h2>Customizing the image<a class="headerlink" href="#customizing-the-image" title="Permalink to this headline">¶</a></h2>
-<p>Customizing the image is an optimized way of adding your own dependencies to the image - better
-suited to prepare highly optimized (for size) production images, especially when you have dependencies
-that require to be compiled before installing (such as <code class="docutils literal notranslate"><span class="pre">mpi4py</span></code>).</p>
-<p>It also allows more sophisticated usages, needed by "Power-users" - for example using forked version
-of Airflow, or building the images from security-vetted sources.</p>
-<p>The big advantage of this method is that it produces optimized image even if you need some compile-time
-dependencies that are not needed in the final image.</p>
-<p>The disadvantage is that you need to use Airflow Sources to build such images from the
-<a class="reference external" href="https://downloads.apache.org/airflow/">official distribution repository of Apache Airflow</a> for the
-released versions, or from the checked out sources (using release tags or main branches) in the
-<a class="reference external" href="https://github.com/apache/airflow">Airflow GitHub Project</a> or from your own fork
-if you happen to do maintain your own fork of Airflow.</p>
-<p>Another disadvantage is that the pattern of building Docker images with <code class="docutils literal notranslate"><span class="pre">--build-arg</span></code> is less familiar
-to developers of such images. However it is quite well-known to "power-users". That's why the
-customizing flow is better suited for those users who have more familiarity and have more custom
-requirements.</p>
-<p>The image also usually builds much longer than the equivalent "Extended" image because instead of
-extending the layers that are already coming from the base image, it rebuilds the layers needed
-to add extra dependencies needed at early stages of image building.</p>
-<p>When customizing the image you can choose a number of options how you install Airflow:</p>
-<blockquote>
-<div><ul class="simple">
-<li><p>From the PyPI releases (default)</p></li>
-<li><p>From the custom installation sources - using additional/replacing the original apt or PyPI repositories</p></li>
-<li><p>From local sources. This is used mostly during development.</p></li>
-<li><p>From tag or branch, or specific commit from a GitHub Airflow repository (or fork). This is particularly
-useful when you build image for a custom version of Airflow that you keep in your fork and you do not
-want to release the custom Airflow version to PyPI.</p></li>
-<li><p>From locally stored binary packages for Airflow, Airflow Providers and other dependencies. This is
-particularly useful if you want to build Airflow in a highly-secure environment where all such packages
-must be vetted by your security team and stored in your private artifact registry. This also
-allows to build airflow image in an air-gaped environment.</p></li>
-<li><p>Side note. Building <code class="docutils literal notranslate"><span class="pre">Airflow</span></code> in an <code class="docutils literal notranslate"><span class="pre">air-gaped</span></code> environment sounds pretty funny, doesn't it?</p></li>
-</ul>
-</div></blockquote>
-<p>You can also add a range of customizations while building the image:</p>
-<blockquote>
-<div><ul class="simple">
-<li><p>base python image you use for Airflow</p></li>
-<li><p>version of Airflow to install</p></li>
-<li><p>extras to install for Airflow (or even removing some default extras)</p></li>
-<li><p>additional apt/python dependencies to use while building Airflow (DEV dependencies)</p></li>
-<li><p>additional apt/python dependencies to install for runtime version of Airflow (RUNTIME dependencies)</p></li>
-<li><p>additional commands and variables to set if needed during building or preparing Airflow runtime</p></li>
-<li><p>choosing constraint file to use when installing Airflow</p></li>
-</ul>
-</div></blockquote>
-<p>Additional explanation is needed for the last point. Airflow uses constraints to make sure
-that it can be predictably installed, even if some new versions of Airflow dependencies are
-released (or even dependencies of our dependencies!). The docker image and accompanying scripts
-usually determine automatically the right versions of constraints to be used based on the Airflow
-version installed and Python version. For example 2.0.2 version of Airflow installed from PyPI
-uses constraints from <code class="docutils literal notranslate"><span class="pre">constraints-2.0.2</span></code> tag). However in some cases - when installing airflow from
-GitHub for example - you have to manually specify the version of constraints used, otherwise
-it will default to the latest version of the constraints which might not be compatible with the
-version of Airflow you use.</p>
-<p>You can also download any version of Airflow constraints and adapt it with your own set of
-constraints and manually set your own versions of dependencies in your own constraints and use the version
-of constraints that you manually prepared.</p>
-<p>You can read more about constraints in the documentation of the
-<a class="reference external" href="http://airflow.apache.org/docs/apache-airflow/stable/installation.html#constraints-files">Installation</a></p>
-</div>
-<div class="section" id="examples-of-image-customizing">
-<h2>Examples of image customizing<a class="headerlink" href="#examples-of-image-customizing" title="Permalink to this headline">¶</a></h2>
-<div class="section" id="building-from-pypi-packages">
-<span id="image-build-pypi"></span><h3>Building from PyPI packages<a class="headerlink" href="#building-from-pypi-packages" title="Permalink to this headline">¶</a></h3>
-<p>This is the basic way of building the custom images from sources.</p>
-<p>The following example builds the production image in version <code class="docutils literal notranslate"><span class="pre">3.6</span></code> with latest PyPI-released Airflow,
-with default set of Airflow extras and dependencies. The <code class="docutils literal notranslate"><span class="pre">2.0.2</span></code> constraints are used automatically.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/stable-airflow.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-<p>The following example builds the production image in version <code class="docutils literal notranslate"><span class="pre">3.7</span></code> with default extras from <code class="docutils literal notranslate"><span class="pre">2.0.2</span></code> PyPI
-package. The <code class="docutils literal notranslate"><span class="pre">2.0.2</span></code> constraints are used automatically.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/pypi-selected-version.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.7-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_VERSION</span><span class="o">=</span><span class="s2">"2.0.2"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-<p>The following example builds the production image in version <code class="docutils literal notranslate"><span class="pre">3.8</span></code> with additional airflow extras
-(<code class="docutils literal notranslate"><span class="pre">mssql,hdfs</span></code>) from <code class="docutils literal notranslate"><span class="pre">2.0.2</span></code> PyPI package, and additional dependency (<code class="docutils literal notranslate"><span class="pre">oauth2client</span></code>).</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/pypi-extras-and-deps.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.8-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_VERSION</span><span class="o">=</span><span class="s2">"2.0.2"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_AIRFLOW_EXTRAS</span><span class="o">=</span><span class="s2">"mssql,hdfs"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_PYTHON_DEPS</span><span class="o">=</span><span class="s2">"oauth2client"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-<p>The following example adds <code class="docutils literal notranslate"><span class="pre">mpi4py</span></code> package which requires both <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> and <code class="docutils literal notranslate"><span class="pre">mpi</span> <span class="pre">compiler</span></code>.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/add-build-essential-custom.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.6-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_VERSION</span><span class="o">=</span><span class="s2">"2.0.2"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_PYTHON_DEPS</span><span class="o">=</span><span class="s2">"mpi4py"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_DEV_APT_DEPS</span><span class="o">=</span><span class="s2">"libopenmpi-dev"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_RUNTIME_APT_DEPS</span><span class="o">=</span><span class="s2">"openmpi-common"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-<p>The above image is equivalent of the "extended" image from previous chapter but it's size is only
-874 MB. Comparing to 1.1 GB of the "extended image" this is about 230 MB less, so you can achieve ~20%
-improvement in size of the image by using "customization" vs. extension. The saving can increase in case you
-have more complex dependencies to build.</p>
-</div>
-<div class="section" id="building-optimized-images">
-<span id="image-build-optimized"></span><h3>Building optimized images<a class="headerlink" href="#building-optimized-images" title="Permalink to this headline">¶</a></h3>
-<p>The following example the production image in version <code class="docutils literal notranslate"><span class="pre">3.6</span></code> with additional airflow extras from <code class="docutils literal notranslate"><span class="pre">2.0.2</span></code>
-PyPI package but it includes additional apt dev and runtime dependencies.</p>
-<p>The dev dependencies are those that require <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> and usually need to involve recompiling
-of some python dependencies so those packages might require some additional DEV dependencies to be
-present during recompilation. Those packages are not needed at runtime, so we only install them for the
-"build" time. They are not installed in the final image, thus producing much smaller images.
-In this case pandas requires recompilation so it also needs gcc and g++ as dev APT dependencies.
-The <code class="docutils literal notranslate"><span class="pre">jre-headless</span></code> does not require recompiling so it can be installed as the runtime APT dependency.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/pypi-dev-runtime-deps.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.6-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_VERSION</span><span class="o">=</span><span class="s2">"2.0.2"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_AIRFLOW_EXTRAS</span><span class="o">=</span><span class="s2">"jdbc"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_PYTHON_DEPS</span><span class="o">=</span><span class="s2">"pandas"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_DEV_APT_DEPS</span><span class="o">=</span><span class="s2">"gcc g++"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_RUNTIME_APT_DEPS</span><span class="o">=</span><span class="s2">"default-jre-headless"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-</div>
-<div class="section" id="building-from-github">
-<span id="image-build-github"></span><h3>Building from GitHub<a class="headerlink" href="#building-from-github" title="Permalink to this headline">¶</a></h3>
-<p>This method is usually used for development purpose. But in case you have your own fork you can point
-it to your forked version of source code without having to release it to PyPI. It is enough to have
-a branch or tag in your repository and use the tag or branch in the URL that you point the installation to.</p>
-<p>In case of GitHyb builds you need to pass the constraints reference manually in case you want to use
-specific constraints, otherwise the default <code class="docutils literal notranslate"><span class="pre">constraints-master</span></code> is used.</p>
-<p>The following example builds the production image in version <code class="docutils literal notranslate"><span class="pre">3.7</span></code> with default extras from the latest master version and
-constraints are taken from latest version of the constraints-master branch in GitHub.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/github-master.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.7-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_INSTALLATION_METHOD</span><span class="o">=</span><span class="s2">"https://github.com/apache/airflow/archive/master.tar.gz#egg=apache-airflow"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_CONSTRAINTS_REFERENCE</span><span class="o">=</span><span class="s2">"constraints-master"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-<p>The following example builds the production image with default extras from the
-latest <code class="docutils literal notranslate"><span class="pre">v2-0-test</span></code> version and constraints are taken from the latest version of
-the <code class="docutils literal notranslate"><span class="pre">constraints-2-0</span></code> branch in GitHub. Note that this command might fail occasionally as only
-the "released version" constraints when building a version and "master" constraints when building
-master are guaranteed to work.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/github-v2-0-test.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.8-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_INSTALLATION_METHOD</span><span class="o">=</span><span class="s2">"https://github.com/apache/airflow/archive/v2-0-test.tar.gz#egg=apache-airflow"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_CONSTRAINTS_REFERENCE</span><span class="o">=</span><span class="s2">"constraints-2-0"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-<p>You can also specify another repository to build from. If you also want to use different constraints
-repository source, you must specify it as additional <code class="docutils literal notranslate"><span class="pre">CONSTRAINTS_GITHUB_REPOSITORY</span></code> build arg.</p>
-<p>The following example builds the production image using <code class="docutils literal notranslate"><span class="pre">potiuk/airflow</span></code> fork of Airflow and constraints
-are also downloaded from that repository.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/github-different-repository.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.8-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_INSTALLATION_METHOD</span><span class="o">=</span><span class="s2">"https://github.com/potiuk/airflow/archive/master.tar.gz#egg=apache-airflow"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_CONSTRAINTS_REFERENCE</span><span class="o">=</span><span class="s2">"constraints-master"</span> <span class="se">\</span>
- --build-arg <span class="nv">CONSTRAINTS_GITHUB_REPOSITORY</span><span class="o">=</span><span class="s2">"potiuk/airflow"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-</div>
-<div class="section" id="using-custom-installation-sources">
-<span id="image-build-custom"></span><h3>Using custom installation sources<a class="headerlink" href="#using-custom-installation-sources" title="Permalink to this headline">¶</a></h3>
-<p>You can customize more aspects of the image - such as additional commands executed before apt dependencies
-are installed, or adding extra sources to install your dependencies from. You can see all the arguments
-described below but here is an example of rather complex command to customize the image
-based on example in <a class="reference external" href="https://github.com/apache/airflow/issues/8605#issuecomment-690065621">this comment</a>:</p>
-<p>In case you need to use your custom PyPI package indexes, you can also customize PYPI sources used during
-image build by adding a <code class="docutils literal notranslate"><span class="pre">docker-context-files</span></code>/<code class="docutils literal notranslate"><span class="pre">.pypirc</span></code> file when building the image.
-This <code class="docutils literal notranslate"><span class="pre">.pypirc</span></code> will not be committed to the repository (it is added to <code class="docutils literal notranslate"><span class="pre">.gitignore</span></code>) and it will not be
-present in the final production image. It is added and used only in the build segment of the image.
-Therefore this <code class="docutils literal notranslate"><span class="pre">.pypirc</span></code> file can safely contain list of package indexes you want to use,
-usernames and passwords used for authentication. More details about <code class="docutils literal notranslate"><span class="pre">.pypirc</span></code> file can be found in the
-<a class="reference external" href="https://packaging.python.org/specifications/pypirc/">pypirc specification</a>.</p>
-<p>Such customizations are independent of the way how airflow is installed.</p>
-<div class="admonition note">
-<p class="admonition-title">Note</p>
-<p>Similar results could be achieved by modifying the Dockerfile manually (see below) and injecting the
-commands needed, but by specifying the customizations via build-args, you avoid the need of
-synchronizing the changes from future Airflow Dockerfiles. Those customizations should work with the
-future version of Airflow's official <code class="docutils literal notranslate"><span class="pre">Dockerfile</span></code> at most with minimal modifications od parameter
-names (if any), so using the build command for your customizations makes your custom image more
-future-proof.</p>
-</div>
-<p>The following - rather complex - example shows capabilities of:</p>
-<blockquote>
-<div><ul class="simple">
-<li><p>Adding airflow extras (slack, odbc)</p></li>
-<li><p>Adding PyPI dependencies (<code class="docutils literal notranslate"><span class="pre">azure-storage-blob,</span> <span class="pre">oauth2client,</span> <span class="pre">beautifulsoup4,</span> <span class="pre">dateparser,</span> <span class="pre">rocketchat_API,typeform</span></code>)</p></li>
-<li><p>Adding custom environment variables while installing <code class="docutils literal notranslate"><span class="pre">apt</span></code> dependencies - both DEV and RUNTIME
-(<code class="docutils literal notranslate"><span class="pre">ACCEPT_EULA=Y'</span></code>)</p></li>
-<li><p>Adding custom curl command for adding keys and configuring additional apt sources needed to install
-<code class="docutils literal notranslate"><span class="pre">apt</span></code> dependencies (both DEV and RUNTIME)</p></li>
-<li><p>Adding custom <code class="docutils literal notranslate"><span class="pre">apt</span></code> dependencies, both DEV (<code class="docutils literal notranslate"><span class="pre">msodbcsql17</span> <span class="pre">unixodbc-dev</span> <span class="pre">g++)</span> <span class="pre">and</span> <span class="pre">runtime</span> <span class="pre">msodbcsql17</span> <span class="pre">unixodbc</span> <span class="pre">git</span> <span class="pre">procps</span> <span class="pre">vim</spa [...]
-</ul>
-</div></blockquote>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/custom-sources.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . -f Dockerfile <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.7-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_VERSION</span><span class="o">=</span><span class="s2">"2.0.2"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_AIRFLOW_EXTRAS</span><span class="o">=</span><span class="s2">"slack,odbc"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_PYTHON_DEPS</span><span class="o">=</span><span class="s2">" \</span>
-<span class="s2"> azure-storage-blob \</span>
-<span class="s2"> oauth2client \</span>
-<span class="s2"> beautifulsoup4 \</span>
-<span class="s2"> dateparser \</span>
-<span class="s2"> rocketchat_API \</span>
-<span class="s2"> typeform"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_DEV_APT_COMMAND</span><span class="o">=</span><span class="s2">"curl https://packages.microsoft.com/keys/microsoft.asc | \</span>
-<span class="s2"> apt-key add --no-tty - && \</span>
-<span class="s2"> curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_DEV_APT_ENV</span><span class="o">=</span><span class="s2">"ACCEPT_EULA=Y"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_DEV_APT_DEPS</span><span class="o">=</span><span class="s2">"msodbcsql17 unixodbc-dev g++"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_RUNTIME_APT_COMMAND</span><span class="o">=</span><span class="s2">"curl https://packages.microsoft.com/keys/microsoft.asc | \</span>
-<span class="s2"> apt-key add --no-tty - && \</span>
-<span class="s2"> curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_RUNTIME_APT_ENV</span><span class="o">=</span><span class="s2">"ACCEPT_EULA=Y"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_RUNTIME_APT_DEPS</span><span class="o">=</span><span class="s2">"msodbcsql17 unixodbc git procps vim"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-</div>
-<div class="section" id="build-images-in-security-restricted-environments">
-<span id="image-build-secure-environments"></span><h3>Build images in security restricted environments<a class="headerlink" href="#build-images-in-security-restricted-environments" title="Permalink to this headline">¶</a></h3>
-<p>You can also make sure your image is only build using local constraint file and locally downloaded
-wheel files. This is often useful in Enterprise environments where the binary files are verified and
-vetted by the security teams. It is also the most complex way of building the image. You should be an
-expert of building and using Dockerfiles in order to use it and have to have specific needs of security if
-you want to follow that route.</p>
-<p>This builds below builds the production image with packages and constraints used from the local
-<code class="docutils literal notranslate"><span class="pre">docker-context-files</span></code> rather than installed from PyPI or GitHub. It also disables MySQL client
-installation as it is using external installation method.</p>
-<p>Note that as a prerequisite - you need to have downloaded wheel files. In the example below we
-first download such constraint file locally and then use <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">download</span></code> to get the <code class="docutils literal notranslate"><span class="pre">.whl</span></code> files needed
-but in most likely scenario, those wheel files should be copied from an internal repository of such .whl
-files. Note that <code class="docutils literal notranslate"><span class="pre">AIRFLOW_VERSION_SPECIFICATION</span></code> is only there for reference, the apache airflow <code class="docutils literal notranslate"><span class="pre">.whl</span></code> file
-in the right version is part of the <code class="docutils literal notranslate"><span class="pre">.whl</span></code> files downloaded.</p>
-<p>Note that 'pip download' will only works on Linux host as some of the packages need to be compiled from
-sources and you cannot install them providing <code class="docutils literal notranslate"><span class="pre">--platform</span></code> switch. They also need to be downloaded using
-the same python version as the target image.</p>
-<p>The <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">download</span></code> might happen in a separate environment. The files can be committed to a separate
-binary repository and vetted/verified by the security team and used subsequently to build images
-of Airflow when needed on an air-gaped system.</p>
-<p>Example of preparing the constraint files and wheel files. Note that <code class="docutils literal notranslate"><span class="pre">mysql</span></code> dependency is removed
-as <code class="docutils literal notranslate"><span class="pre">mysqlclient</span></code> is installed from Oracle's <code class="docutils literal notranslate"><span class="pre">apt</span></code> repository and if you want to add it, you need
-to provide this library from you repository if you want to build Airflow image in an "air-gaped" system.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/restricted/restricted_environments.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>rm docker-context-files/*.whl docker-context-files/*.tar.gz docker-context-files/*.txt <span class="o">||</span> <span class="nb">true</span>
-
-curl -Lo <span class="s2">"docker-context-files/constraints-3.7.txt"</span> <span class="se">\</span>
- https://raw.githubusercontent.com/apache/airflow/constraints-2.0.2/constraints-3.7.txt
-
-pip download --dest docker-context-files <span class="se">\</span>
- --constraint docker-context-files/constraints-3.7.txt <span class="se">\</span>
- <span class="s2">"apache-airflow[async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,postgres,redis,slack,ssh,statsd,virtualenv]==2.0.2"</span>
-</pre></div>
-</div>
-</div>
-<p>After this step is finished, your <code class="docutils literal notranslate"><span class="pre">docker-context-files</span></code> folder will contain all the packages that
-are needed to install Airflow from.</p>
-<p>Those downloaded packages and constraint file can be pre-vetted by your security team before you attempt
-to install the image. You can also store those downloaded binary packages in your private artifact registry
-which allows for the flow where you will download the packages on one machine, submit only new packages for
-security vetting and only use the new packages when they were vetted.</p>
-<p>On a separate (air-gaped) system, all the PyPI packages can be copied to <code class="docutils literal notranslate"><span class="pre">docker-context-files</span></code>
-where you can build the image using the packages downloaded by passing those build args:</p>
-<blockquote>
-<div><ul class="simple">
-<li><p><code class="docutils literal notranslate"><span class="pre">INSTALL_FROM_DOCKER_CONTEXT_FILES="true"</span></code> - to use packages present in <code class="docutils literal notranslate"><span class="pre">docker-context-files</span></code></p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW_PRE_CACHED_PIP_PACKAGES="false"</span></code> - to not pre-cache packages from PyPI when building image</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW_CONSTRAINTS_LOCATION=/docker-context-files/YOUR_CONSTRAINT_FILE.txt</span></code> - to downloaded constraint files</p></li>
-<li><p>(Optional) <code class="docutils literal notranslate"><span class="pre">INSTALL_MYSQL_CLIENT="false"</span></code> if you do not want to install <code class="docutils literal notranslate"><span class="pre">MySQL</span></code>
-client from the Oracle repositories. In this case also make sure that your</p></li>
-</ul>
-</div></blockquote>
-<p>Note, that the solution we have for installing python packages from local packages, only solves the problem
-of "air-gaped" python installation. The Docker image also downloads <code class="docutils literal notranslate"><span class="pre">apt</span></code> dependencies and <code class="docutils literal notranslate"><span class="pre">node-modules</span></code>.
-Those type of dependencies are however more likely to be available in your "air-gaped" system via transparent
-proxies and it should automatically reach out to your private registries, however in the future the
-solution might be applied to both of those installation steps.</p>
-<p>You can also use techniques described in the previous chapter to make <code class="docutils literal notranslate"><span class="pre">docker</span> <span class="pre">build</span></code> use your private
-apt sources or private PyPI repositories (via <code class="docutils literal notranslate"><span class="pre">.pypirc</span></code>) available which can be security-vetted.</p>
-<p>If you fulfill all the criteria, you can build the image on an air-gaped system by running command similar
-to the below:</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/restricted/restricted_environments.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.7-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_INSTALLATION_METHOD</span><span class="o">=</span><span class="s2">"apache-airflow"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_VERSION</span><span class="o">=</span><span class="s2">"2.0.2"</span> <span class="se">\</span>
- --build-arg <span class="nv">INSTALL_MYSQL_CLIENT</span><span class="o">=</span><span class="s2">"false"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_PRE_CACHED_PIP_PACKAGES</span><span class="o">=</span><span class="s2">"false"</span> <span class="se">\</span>
- --build-arg <span class="nv">INSTALL_FROM_DOCKER_CONTEXT_FILES</span><span class="o">=</span><span class="s2">"true"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_CONSTRAINTS_LOCATION</span><span class="o">=</span><span class="s2">"/docker-context-files/constraints-3.7.txt"</span>
-</pre></div>
-</div>
-</div>
-</div>
-<div class="section" id="modifying-the-dockerfile">
-<h3>Modifying the Dockerfile<a class="headerlink" href="#modifying-the-dockerfile" title="Permalink to this headline">¶</a></h3>
-<p>The build arg approach is a convenience method if you do not want to manually modify the <code class="docutils literal notranslate"><span class="pre">Dockerfile</span></code>.
-Our approach is flexible enough, to be able to accommodate most requirements and
-customizations out-of-the-box. When you use it, you do not need to worry about adapting the image every
-time new version of Airflow is released. However sometimes it is not enough if you have very
-specific needs and want to build a very custom image. In such case you can simply modify the
-<code class="docutils literal notranslate"><span class="pre">Dockerfile</span></code> manually as you see fit and store it in your forked repository. However you will have to
-make sure to rebase your changes whenever new version of Airflow is released, because we might modify
-the approach of our Dockerfile builds in the future and you might need to resolve conflicts
-and rebase your changes.</p>
-<p>There are a few things to remember when you modify the <code class="docutils literal notranslate"><span class="pre">Dockerfile</span></code>:</p>
-<ul>
-<li><p>We are using the widely recommended pattern of <code class="docutils literal notranslate"><span class="pre">.dockerignore</span></code> where everything is ignored by default
-and only the required folders are added through exclusion (!). This allows to keep docker context small
-because there are many binary artifacts generated in the sources of Airflow and if they are added to
-the context, the time of building the image would increase significantly. If you want to add any new
-folders to be available in the image you must add it here with leading <code class="docutils literal notranslate"><span class="pre">!</span></code>.</p>
-<div class="highlight-text notranslate"><div class="highlight"><pre><span></span># Ignore everything
-**
-
-# Allow only these directories
-!airflow
-...
-</pre></div>
-</div>
-</li>
-<li><p>The <code class="docutils literal notranslate"><span class="pre">docker-context-files</span></code> folder is automatically added to the context of the image, so if you want
-to add individual files, binaries, requirement files etc you can add them there. The
-<code class="docutils literal notranslate"><span class="pre">docker-context-files</span></code> is copied to the <code class="docutils literal notranslate"><span class="pre">/docker-context-files</span></code> folder of the build segment of the
-image, so it is not present in the final image - which makes the final image smaller in case you want
-to use those files only in the <code class="docutils literal notranslate"><span class="pre">build</span></code> segment. You must copy any files from the directory manually,
-using COPY command if you want to get the files in your final image (in the main image segment).</p></li>
+ <div class="section" id="module-airflow.executors.celery_executor">
+<span id="airflow-executors-celery-executor"></span><h1><a class="reference internal" href="#module-airflow.executors.celery_executor" title="airflow.executors.celery_executor"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a><a class="headerlink" href="#module-airflow.executors.celery_executor" title="Permalink to this headline">¶</a></h1>
+<p><p>CeleryExecutor</p>
+<div class="admonition seealso">
+<p class="admonition-title">See also</p>
+<p>For more information on how the CeleryExecutor works, take a look at the guide:
+<a class="reference internal" href="../../../../executor/celery.html#executor-celeryexecutor"><span class="std std-ref">Celery Executor</span></a></p>
+</div>
+</p>
+<div class="section" id="module-contents">
+<h2>Module Contents<a class="headerlink" href="#module-contents" title="Permalink to this headline">¶</a></h2>
+<dl class="py data">
+<dt id="airflow.executors.celery_executor.log">
+<code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">log</code><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#log"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.log" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.celery_executor.CELERY_FETCH_ERR_MSG_HEADER">
+<code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">CELERY_FETCH_ERR_MSG_HEADER</code><em class="property"> = Error fetching Celery task state</em><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#CELERY_FETCH_ERR_MSG_HEADER"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.CELERY_FETCH_ERR_MSG_HEADER" title="Permalink to [...]
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.celery_executor.CELERY_SEND_ERR_MSG_HEADER">
+<code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">CELERY_SEND_ERR_MSG_HEADER</code><em class="property"> = Error sending Celery task</em><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#CELERY_SEND_ERR_MSG_HEADER"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.CELERY_SEND_ERR_MSG_HEADER" title="Permalink to this defin [...]
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.celery_executor.OPERATION_TIMEOUT">
+<code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">OPERATION_TIMEOUT</code><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#OPERATION_TIMEOUT"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.OPERATION_TIMEOUT" title="Permalink to this definition">¶</a></dt>
+<dd><p>To start the celery worker, run the command:
+airflow celery worker</p>
+</dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.celery_executor.celery_configuration">
+<code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">celery_configuration</code><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#celery_configuration"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.celery_configuration" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.celery_executor.app">
+<code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">app</code><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#app"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.app" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py function">
+<dt id="airflow.executors.celery_executor.execute_command">
+<code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">execute_command</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">command_to_exec</span><span class="p">:</span> <span class="n">CommandType</span></em><span class="sig-paren">)</span> → <a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.9)">None</a><a class="reference internal" href= [...]
+<dt>
+<code class="sig-name descname">Executes command.</code></dt>
+<dd></dd></dl>
+
+<dl class="py function">
+<dt id="airflow.executors.celery_executor._execute_in_fork">
+<code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">_execute_in_fork</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">command_to_exec</span><span class="p">:</span> <span class="n">CommandType</span></em><span class="sig-paren">)</span> → <a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.9)">None</a><a class="reference internal" href [...]
+<dd></dd></dl>
+
+<dl class="py function">
+<dt id="airflow.executors.celery_executor._execute_in_subprocess">
+<code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">_execute_in_subprocess</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">command_to_exec</span><span class="p">:</span> <span class="n">CommandType</span></em><span class="sig-paren">)</span> → <a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.9)">None</a><a class="reference internal [...]
+<dd></dd></dl>
+
+<dl class="py class">
+<dt id="airflow.executors.celery_executor.ExceptionWithTraceback">
+<em class="property">class </em><code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">ExceptionWithTraceback</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">exception</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/exceptions.html#Exception" title="(in Python v3.9)">Exception</a></span></em>, <em class="sig-param"><span class="n">exce [...]
+<dd><p>Wrapper class used to propagate exceptions to parent processes from subprocesses.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>exception</strong> (<a class="reference external" href="https://docs.python.org/3/library/exceptions.html#Exception" title="(in Python v3.9)"><em>Exception</em></a>) -- The exception to wrap</p></li>
+<li><p><strong>exception_traceback</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)"><em>str</em></a>) -- The stacktrace to wrap</p></li>
</ul>
-</div>
-</div>
-<div class="section" id="more-details">
-<h2>More details<a class="headerlink" href="#more-details" title="Permalink to this headline">¶</a></h2>
-<div class="section" id="build-args-reference">
-<h3>Build Args reference<a class="headerlink" href="#build-args-reference" title="Permalink to this headline">¶</a></h3>
-<p>The detailed <code class="docutils literal notranslate"><span class="pre">--build-arg</span></code> reference can be found in <a class="reference internal" href="build-arg-ref.html"><span class="doc">Image build arguments reference</span></a>.</p>
-</div>
-<div class="section" id="the-architecture-of-the-images">
-<h3>The architecture of the images<a class="headerlink" href="#the-architecture-of-the-images" title="Permalink to this headline">¶</a></h3>
-<p>You can read more details about the images - the context, their parameters and internal structure in the
-<a class="reference external" href="https://github.com/apache/airflow/blob/master/IMAGES.rst">IMAGES.rst</a> document.</p>
-</div>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.celery_executor.TaskInstanceInCelery">
+<code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">TaskInstanceInCelery</code><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#TaskInstanceInCelery"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.TaskInstanceInCelery" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py function">
+<dt id="airflow.executors.celery_executor.send_task_to_executor">
+<code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">send_task_to_executor</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">task_tuple</span><span class="p">:</span> <span class="n"><a class="reference internal" href="#airflow.executors.celery_executor.TaskInstanceInCelery" title="airflow.executors.celery_executor.TaskInstanceInCelery">TaskInstanceInCelery</a></span></em><span class="sig-paren">)</span [...]
+<dt>
+<code class="sig-name descname">Sends task to executor.</code></dt>
+<dd></dd></dl>
+
+<dl class="py function">
+<dt id="airflow.executors.celery_executor.on_celery_import_modules">
+<code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">on_celery_import_modules</code><span class="sig-paren">(</span><em class="sig-param"><span class="o">*</span><span class="n">args</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#on_celery_import_modules">< [...]
+<dt>
+<code class="sig-name descname">Preload some "expensive" airflow modules so that every task process doesn't have to import it again and</code></dt>
+<dt>
+<code class="sig-name descname">again.</code></dt>
+<dd><p>Loading these for each task adds 0.3-0.5s <em>per task</em> before the task can run. For long running tasks this
+doesn't matter, but for short tasks this starts to be a noticeable impact.</p>
+</dd></dl>
+
+<dl class="py class">
+<dt id="airflow.executors.celery_executor.CeleryExecutor">
+<em class="property">class </em><code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">CeleryExecutor</code><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#CeleryExecutor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.CeleryExecutor" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="../base_executor/index.html#airflow.executors.base_executor.BaseExecutor" title="airflow.executors.base_executor.BaseExecutor"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.executors.base_executor.BaseExecutor</span></code></a></p>
+<p>CeleryExecutor is recommended for production use of Airflow. It allows
+distributing the execution of task instances to multiple worker nodes.</p>
+<p>Celery is a simple, flexible and reliable distributed system to process
+vast amounts of messages, while providing operations with the tools
+required to maintain such a system.</p>
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor.start">
+<code class="sig-name descname">start</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#CeleryExecutor.start"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.CeleryExecutor.start" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor._num_tasks_per_send_process">
+<code class="sig-name descname">_num_tasks_per_send_process</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">to_send_count</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.9)">int</a></span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executor [...]
+<dd><p>How many Celery tasks should each worker process send.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Number of tasks that should be sent per process</p>
+</dd>
+<dt class="field-even">Return type</dt>
+<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.9)">int</a></p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor.trigger_tasks">
+<code class="sig-name descname">trigger_tasks</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">open_slots</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.9)">int</a></span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor [...]
+<dd><p>Overwrite trigger_tasks function from BaseExecutor</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>open_slots</strong> -- Number of open slots</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p></p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor._process_tasks">
+<code class="sig-name descname">_process_tasks</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">task_tuples_to_send</span><span class="p">:</span> <span class="n">List<span class="p">[</span><a class="reference internal" href="#airflow.executors.celery_executor.TaskInstanceInCelery" title="airflow.executors.celery_executor.TaskInstanceInCelery">TaskInstanceInCelery</a><span class="p">]</span></span></em><s [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor._send_tasks_to_celery">
+<code class="sig-name descname">_send_tasks_to_celery</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">task_tuples_to_send</span><span class="p">:</span> <span class="n">List<span class="p">[</span><a class="reference internal" href="#airflow.executors.celery_executor.TaskInstanceInCelery" title="airflow.executors.celery_executor.TaskInstanceInCelery">TaskInstanceInCelery</a><span class="p">]</span></span> [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor.sync">
+<code class="sig-name descname">sync</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#CeleryExecutor.sync"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.CeleryExecutor.sync" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor._check_for_stalled_adopted_tasks">
+<code class="sig-name descname">_check_for_stalled_adopted_tasks</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#CeleryExecutor._check_for_stalled_adopted_tasks"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.CeleryExecutor._check_for_stalled_adopted_tasks" [...]
+<dd><p>See if any of the tasks we adopted from another Executor run have not
+progressed after the configured timeout.</p>
+<p>If they haven't, they likely never made it to Celery, and we should
+just resend them. We do that by clearing the state and letting the
+normal scheduler loop deal with that</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor.debug_dump">
+<code class="sig-name descname">debug_dump</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#CeleryExecutor.debug_dump"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.CeleryExecutor.debug_dump" title="Permalink to this definition">¶</a></dt>
+<dd><p>Called in response to SIGUSR2 by the scheduler</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor.update_all_task_states">
+<code class="sig-name descname">update_all_task_states</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#CeleryExecutor.update_all_task_states"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.CeleryExecutor.update_all_task_states" title="Permalink to this defi [...]
+<dd><p>Updates states of the tasks.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor.change_state">
+<code class="sig-name descname">change_state</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">key</span><span class="p">:</span> <span class="n">TaskInstanceKey</span></em>, <em class="sig-param"><span class="n">state</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a></span></em>, <em class [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor.update_task_state">
+<code class="sig-name descname">update_task_state</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">key</span><span class="p">:</span> <span class="n">TaskInstanceKey</span></em>, <em class="sig-param"><span class="n">state</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a></span></em>, <em [...]
+<dd><p>Updates state of a single task.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor.end">
+<code class="sig-name descname">end</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">synchronous</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.9)">bool</a></span> <span class="o">=</span> <span class="default_value">False</span></em><span class="sig-paren">)</span><a class="reference internal" h [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor.execute_async">
+<code class="sig-name descname">execute_async</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">key</span><span class="p">:</span> <span class="n">TaskInstanceKey</span></em>, <em class="sig-param"><span class="n">command</span><span class="p">:</span> <span class="n">CommandType</span></em>, <em class="sig-param"><span class="n">queue</span><span class="p">:</span> <span class="n">Optional<span class="p">[ [...]
+<dd><p>Do not allow async execution for Celery executor.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor.terminate">
+<code class="sig-name descname">terminate</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#CeleryExecutor.terminate"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.CeleryExecutor.terminate" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.CeleryExecutor.try_adopt_task_instances">
+<code class="sig-name descname">try_adopt_task_instances</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">tis</span><span class="p">:</span> <span class="n">List<span class="p">[</span>TaskInstance<span class="p">]</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#CeleryExecutor.try_adopt_task_instances"><span c [...]
+<dd></dd></dl>
+
+</dd></dl>
+
+<dl class="py function">
+<dt id="airflow.executors.celery_executor.fetch_celery_task_state">
+<code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">fetch_celery_task_state</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">async_result</span><span class="p">:</span> <span class="n">AsyncResult</span></em><span class="sig-paren">)</span> → Tuple<span class="p">[</span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a><spa [...]
+<dt>
+<code class="sig-name descname">Fetch and return the state of the given celery task. The scope of this function is</code></dt>
+<dt>
+<code class="sig-name descname">global so that it can be called by subprocesses in the pool.</code></dt>
+<dd><dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>async_result</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.9)"><em>tuple</em></a><em>(</em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.celeryproject.org/en/stable/reference/celery.result.html#celery.result.AsyncResult" title="(in Celery [...]
+to fetch the task's state</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>a tuple of the Celery task key and the Celery state and the celery info
+of the task</p>
+</dd>
+<dt class="field-odd">Return type</dt>
+<dd class="field-odd"><p><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.9)">tuple</a>[<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a>, <a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a>, <a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title= [...]
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py class">
+<dt id="airflow.executors.celery_executor.BulkStateFetcher">
+<em class="property">class </em><code class="sig-prename descclassname">airflow.executors.celery_executor.</code><code class="sig-name descname">BulkStateFetcher</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">sync_parallelism</span><span class="o">=</span><span class="default_value">None</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#BulkStateFetcher"><span class="view [...]
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.utils.log.logging_mixin.LoggingMixin</span></code></p>
+<p>Gets status for many Celery tasks using the best method available</p>
+<p>If BaseKeyValueStoreBackend is used as result backend, the mget method is used.
+If DatabaseBackend is used as result backend, the SELECT ...WHERE task_id IN (...) query is used
+Otherwise, multiprocessing.Pool will be used. Each task status will be downloaded individually.</p>
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.BulkStateFetcher._tasks_list_to_task_ids">
+<code class="sig-name descname">_tasks_list_to_task_ids</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">async_tasks</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#BulkStateFetcher._tasks_list_to_task_ids"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.B [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.BulkStateFetcher.get_many">
+<code class="sig-name descname">get_many</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">async_results</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#BulkStateFetcher.get_many"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_executor.BulkStateFetcher.get_many" ti [...]
+<dd><p>Gets status for many Celery tasks using the best method available.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.BulkStateFetcher._get_many_from_kv_backend">
+<code class="sig-name descname">_get_many_from_kv_backend</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">async_tasks</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#BulkStateFetcher._get_many_from_kv_backend"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_execut [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.BulkStateFetcher._get_many_from_db_backend">
+<code class="sig-name descname">_get_many_from_db_backend</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">async_tasks</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#BulkStateFetcher._get_many_from_db_backend"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_execut [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.BulkStateFetcher._prepare_state_and_info_by_task_dict">
+<em class="property">static </em><code class="sig-name descname">_prepare_state_and_info_by_task_dict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">task_ids</span></em>, <em class="sig-param"><span class="n">task_results_by_task_id</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#BulkStateFetcher._prepare_state_and_info_by_task_dict"><span class="viewcode-link">[source] [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_executor.BulkStateFetcher._get_many_using_multiprocessing">
+<code class="sig-name descname">_get_many_using_multiprocessing</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">async_results</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_executor.html#BulkStateFetcher._get_many_using_multiprocessing"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors [...]
+<dd></dd></dl>
+
+</dd></dl>
+
</div>
</div>
<div class="pager" role="navigation" aria-label="related navigation">
- <a rel="prev" title="Docker Image for Apache Airflow" href="index.html" >
+ <a rel="prev" title="airflow.executors.base_executor" href="../base_executor/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="p">Previous</button>
</a>
- <a rel="next" title="Entrypoint" href="entrypoint.html" >
+ <a rel="next" title="airflow.executors.celery_kubernetes_executor" href="../celery_kubernetes_executor/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="n">Next</button>
</a>
</div>
@@ -1300,30 +1268,117 @@ using COPY command if you want to get the files in your final image (in the main
<nav class="wy-nav-side-toc">
<div class="wy-menu-vertical">
<ul>
-<li><a class="reference internal" href="#">Building the image</a><ul>
-<li><a class="reference internal" href="#extending-vs-customizing-the-image">Extending vs. customizing the image</a></li>
-<li><a class="reference internal" href="#extending-the-image">Extending the image</a></li>
-<li><a class="reference internal" href="#examples-of-image-extending">Examples of image extending</a><ul>
-<li><a class="reference internal" href="#an-apt-package-example">An <code class="docutils literal notranslate"><span class="pre">apt</span></code> package example</a></li>
-<li><a class="reference internal" href="#a-pypi-package-example">A <code class="docutils literal notranslate"><span class="pre">PyPI</span></code> package example</a></li>
-<li><a class="reference internal" href="#a-umask-requiring-example">A <code class="docutils literal notranslate"><span class="pre">umask</span></code> requiring example</a></li>
-<li><a class="reference internal" href="#a-build-essential-requiring-package-example">A <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> requiring package example</a></li>
-<li><a class="reference internal" href="#dag-embedding-example">DAG embedding example</a></li>
+<li><a class="reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a><ul>
+<li><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.log">log</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CELERY_FETCH_ERR_MSG_HEADER">CELERY_FETCH_ERR_MSG_HEADER</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CELERY_SEND_ERR_MSG_HEADER">CELERY_SEND_ERR_MSG_HEADER</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.OPERATION_TIMEOUT">OPERATION_TIMEOUT</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.celery_configuration">celery_configuration</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.app">app</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.execute_command">execute_command</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor._execute_in_fork">_execute_in_fork</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor._execute_in_subprocess">_execute_in_subprocess</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.ExceptionWithTraceback">ExceptionWithTraceback</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.TaskInstanceInCelery">TaskInstanceInCelery</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.send_task_to_executor">send_task_to_executor</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.on_celery_import_modules">on_celery_import_modules</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor">CeleryExecutor</a><ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.start">start</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor._num_tasks_per_send_process">_num_tasks_per_send_process</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.trigger_tasks">trigger_tasks</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor._process_tasks">_process_tasks</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor._send_tasks_to_celery">_send_tasks_to_celery</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.sync">sync</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor._check_for_stalled_adopted_tasks">_check_for_stalled_adopted_tasks</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.debug_dump">debug_dump</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.update_all_task_states">update_all_task_states</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.change_state">change_state</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.update_task_state">update_task_state</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.end">end</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.execute_async">execute_async</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.terminate">terminate</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.CeleryExecutor.try_adopt_task_instances">try_adopt_task_instances</a></li>
</ul>
</li>
-<li><a class="reference internal" href="#customizing-the-image">Customizing the image</a></li>
-<li><a class="reference internal" href="#examples-of-image-customizing">Examples of image customizing</a><ul>
-<li><a class="reference internal" href="#building-from-pypi-packages">Building from PyPI packages</a></li>
-<li><a class="reference internal" href="#building-optimized-images">Building optimized images</a></li>
-<li><a class="reference internal" href="#building-from-github">Building from GitHub</a></li>
-<li><a class="reference internal" href="#using-custom-installation-sources">Using custom installation sources</a></li>
-<li><a class="reference internal" href="#build-images-in-security-restricted-environments">Build images in security restricted environments</a></li>
-<li><a class="reference internal" href="#modifying-the-dockerfile">Modifying the Dockerfile</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.fetch_celery_task_state">fetch_celery_task_state</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher">BulkStateFetcher</a><ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._tasks_list_to_task_ids">_tasks_list_to_task_ids</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher.get_many">get_many</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._get_many_from_kv_backend">_get_many_from_kv_backend</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._get_many_from_db_backend">_get_many_from_db_backend</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._prepare_state_and_info_by_task_dict">_prepare_state_and_info_by_task_dict</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_executor.BulkStateFetcher._get_many_using_multiprocessing">_get_many_using_multiprocessing</a></li>
</ul>
</li>
-<li><a class="reference internal" href="#more-details">More details</a><ul>
-<li><a class="reference internal" href="#build-args-reference">Build Args reference</a></li>
-<li><a class="reference internal" href="#the-architecture-of-the-images">The architecture of the images</a></li>
</ul>
</li>
</ul>
@@ -1350,7 +1405,7 @@ using COPY command if you want to get the files in your final image (in the main
<div class="base-layout--button">
- <a href="https://github.com/apache/airflow/edit/devel/docs/docker-stack/build.rst" rel="nofollow">
+ <a href="https://github.com/apache/airflow/edit/devel/docs/apache-airflow/_api/airflow/executors/celery_executor/index.rst" rel="nofollow">
<button class="btn-hollow btn-brown btn-with-icon with-box-shadow button-fixed">
<svg xmlns="http://www.w3.org/2000/svg" width="30.76" height="30">
@@ -1484,15 +1539,16 @@ using COPY command if you want to get the files in your final image (in the main
</span>
</div>
</footer>
- <script type="text/javascript" src="_static/_gen/js/docs.js"></script>
- <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/_gen/js/docs.js"></script>
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
<script src="https://code.jquery.com/jquery-3.4.1.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script>
- <script src="_static/jquery.js"></script>
- <script src="_static/underscore.js"></script>
- <script src="_static/doctools.js"></script>
- <script src="_static/clipboard.min.js"></script>
- <script src="_static/copybutton.js"></script>
+ <script src="../../../../_static/jquery.js"></script>
+ <script src="../../../../_static/underscore.js"></script>
+ <script src="../../../../_static/doctools.js"></script>
+ <script src="../../../../_static/clipboard.min.js"></script>
+ <script src="../../../../_static/copybutton.js"></script>
+ <script src="../../../../_static/jira-links.js"></script>
</body>
</html>
\ No newline at end of file
diff --git a/docs-archive/docker-stack/recipes.html b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/celery_kubernetes_executor/index.html
similarity index 63%
copy from docs-archive/docker-stack/recipes.html
copy to docs-archive/apache-airflow/2.1.0/_api/airflow/executors/celery_kubernetes_executor/index.html
index 75019af..985de5b 100644
--- a/docs-archive/docker-stack/recipes.html
+++ b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/celery_kubernetes_executor/index.html
@@ -9,15 +9,16 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Recipes — docker-stack Documentation</title>
- <link rel="stylesheet" href="_static/_gen/css/main.min.css" type="text/css" />
- <link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
- <link rel="stylesheet" type="text/css" href="_static/_gen/css/main-custom.min.css" />
- <link rel="shortcut icon" href="_static/pin_32.png"/>
- <link rel="index" title="Index" href="genindex.html" />
- <link rel="search" title="Search" href="search.html" />
- <link rel="next" title="Image build arguments reference" href="build-arg-ref.html" />
- <link rel="prev" title="Entrypoint" href="entrypoint.html" />
+ <title>airflow.executors.celery_kubernetes_executor — Airflow Documentation</title>
+ <link rel="stylesheet" href="../../../../_static/_gen/css/main.min.css" type="text/css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/copybutton.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/_gen/css/main-custom.min.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/graphviz.css" />
+ <link rel="shortcut icon" href="../../../../_static/pin_32.png"/>
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+ <link rel="next" title="airflow.executors.dask_executor" href="../dask_executor/index.html" />
+ <link rel="prev" title="airflow.executors.celery_executor" href="../celery_executor/index.html" />
<script type="application/javascript">
var doNotTrack = false;
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
@@ -272,7 +273,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -283,7 +284,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -313,19 +314,133 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Recipes</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
-</ul>
-</li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6"><a class="reference internal" href="../base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor">CeleryKubernetesExecutor</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.KUBERNETES_QUEUE">KUBERNETES_QUEUE</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.queued_tasks">queued_tasks</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.running">running</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.job_id">job_id</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.slots_available">slots_available</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.start">start</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.queue_command">queue_command</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.queue_task_instance">queue_task_instance</a></li>
+</ul>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.has_task">has_task</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.heartbeat">heartbeat</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.get_event_buffer">get_event_buffer</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.try_adopt_task_instances">try_adopt_task_instances</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.end">end</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.terminate">terminate</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor._router">_router</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.debug_dump">debug_dump</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l6"><a class="reference internal" href="../dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -409,7 +524,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -420,7 +535,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -450,19 +565,133 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Recipes</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
-</ul>
-</li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6"><a class="reference internal" href="../base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor">CeleryKubernetesExecutor</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.KUBERNETES_QUEUE">KUBERNETES_QUEUE</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.queued_tasks">queued_tasks</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.running">running</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.job_id">job_id</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.slots_available">slots_available</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.start">start</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.queue_command">queue_command</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.queue_task_instance">queue_task_instance</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.has_task">has_task</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.heartbeat">heartbeat</a></li>
+</ul>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.get_event_buffer">get_event_buffer</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.try_adopt_task_instances">try_adopt_task_instances</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.end">end</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.terminate">terminate</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor._router">_router</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.debug_dump">debug_dump</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l6"><a class="reference internal" href="../dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -551,9 +780,13 @@
<ul class="breadcrumb">
- <li class="breadcrumb-item"><a href="index.html" class="icon icon-home"> Home</a></li>
+ <li class="breadcrumb-item"><a href="../../../../index.html" class="icon icon-home"> Home</a></li>
+
+ <li class="breadcrumb-item"><a href="../../../../python-api-ref.html">Python API Reference</a></li>
- <li class="breadcrumb-item"><a href="recipes.html"> Recipes</a></li>
+ <li class="breadcrumb-item"><a href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a></li>
+
+ <li class="breadcrumb-item"><a href="index.html"> <code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
</ul>
</div>
@@ -564,192 +797,163 @@
<div class="bodywrapper">
<div class="body" role="main">
- <blockquote>
-<div></div></blockquote>
-<div class="section" id="recipes">
-<h1>Recipes<a class="headerlink" href="#recipes" title="Permalink to this headline">¶</a></h1>
-<p>Users sometimes share interesting ways of using the Docker images. We encourage users to contribute these
-recipes to the documentation in case they prove useful to other members of the community by
-submitting a pull request. The sections below capture this knowledge.</p>
-<div class="section" id="google-cloud-sdk-installation">
-<h2>Google Cloud SDK installation<a class="headerlink" href="#google-cloud-sdk-installation" title="Permalink to this headline">¶</a></h2>
-<p>Some operators, such as <a class="reference external" href="/docs/apache-airflow-providers-google/stable/_api/airflow/providers/google/cloud/operators/kubernetes_engine/index.html#airflow.providers.google.cloud.operators.kubernetes_engine.GKEStartPodOperator" title="(in apache-airflow-providers-google vdevel)"><code class="xref py py-class docutils literal notranslate"><span class="pre">GKEStartPodOperator</span></code></a>,
-<a class="reference external" href="/docs/apache-airflow-providers-google/stable/_api/airflow/providers/google/cloud/operators/dataflow/index.html#airflow.providers.google.cloud.operators.dataflow.DataflowStartSqlJobOperator" title="(in apache-airflow-providers-google vdevel)"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataflowStartSqlJobOperator</span></code></a>, require
-the installation of <a class="reference external" href="https://cloud.google.com/sdk">Google Cloud SDK</a> (includes <code class="docutils literal notranslate"><span class="pre">gcloud</span></code>).
-You can also run these commands with BashOperator.</p>
-<p>Create a new Dockerfile like the one shown below.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-images-recipes/gcloud.Dockerfile</span></p>
-<div class="highlight-dockerfile notranslate"><div class="highlight"><pre><span></span><span class="c"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
-<span class="c"># contributor license agreements. See the NOTICE file distributed with</span>
-<span class="c"># this work for additional information regarding copyright ownership.</span>
-<span class="c"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
-<span class="c"># (the "License"); you may not use this file except in compliance with</span>
-<span class="c"># the License. You may obtain a copy of the License at</span>
-<span class="c">#</span>
-<span class="c"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c">#</span>
-<span class="c"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c"># See the License for the specific language governing permissions and</span>
-<span class="c"># limitations under the License.</span>
-<span class="k">ARG</span> BASE_AIRFLOW_IMAGE
-<span class="k">FROM</span> <span class="s">${BASE_AIRFLOW_IMAGE}</span>
-
-<span class="k">SHELL</span> <span class="p">[</span><span class="s2">"/bin/bash"</span><span class="p">,</span> <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"pipefail"</span><span class="p">,</span> <span class="s2">"-e"</span><span class="p">,</span> <span class="s2">"-u"</span><span class="p">,</span> <span class="s2">"-x"</span><span class="p">,</span> <span class="s2">"-c"</span><span clas [...]
-
-<span class="k">USER</span><span class="s"> 0</span>
-
-<span class="k">ARG</span> <span class="nv">CLOUD_SDK_VERSION</span><span class="o">=</span><span class="m">322</span>.0.0
-<span class="k">ENV</span> <span class="nv">GCLOUD_HOME</span><span class="o">=</span>/opt/google-cloud-sdk
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">/bin/:</span><span class="si">${</span><span class="nv">PATH</span><span class="si">}</span><span class="s2">"</span>
-
-<span class="k">RUN</span> <span class="nv">DOWNLOAD_URL</span><span class="o">=</span><span class="s2">"https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-</span><span class="si">${</span><span class="nv">CLOUD_SDK_VERSION</span><span class="si">}</span><span class="s2">-linux-x86_64.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="nv">TMP_DIR</span><span class="o">=</span><span class="s2">"</span><span class="k">$(</span>mktemp -d<span class="k">)</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> curl -fL <span class="s2">"</span><span class="si">${</span><span class="nv">DOWNLOAD_URL</span><span class="si">}</span><span class="s2">"</span> --output <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">/google-cloud-sdk.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir -p <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> tar xzf <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">/google-cloud-sdk.tar.gz"</span> -C <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">/install.sh"</span> <span class="se">\</span>
- --bash-completion<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --path-update<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --usage-reporting<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --additional-components alpha beta kubectl <span class="se">\</span>
- --quiet <span class="se">\</span>
- <span class="o">&&</span> rm -rf <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> gcloud --version
-
-<span class="k">USER</span><span class="s"> ${AIRFLOW_UID}</span>
-</pre></div>
-</div>
-</div>
-<p>Then build a new image.</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">BASE_AIRFLOW_IMAGE</span><span class="o">=</span><span class="s2">"apache/airflow:2.0.2"</span> <span class="se">\</span>
- -t my-airflow-image
-</pre></div>
-</div>
-</div>
-<div class="section" id="apache-hadoop-stack-installation">
-<h2>Apache Hadoop Stack installation<a class="headerlink" href="#apache-hadoop-stack-installation" title="Permalink to this headline">¶</a></h2>
-<p>Airflow is often used to run tasks on Hadoop cluster. It required Java Runtime Environment (JRE) to run.
-Below are the steps to take tools that are frequently used in Hadoop-world:</p>
-<ul class="simple">
-<li><p>Java Runtime Environment (JRE)</p></li>
-<li><p>Apache Hadoop</p></li>
-<li><p>Apache Hive</p></li>
-<li><p><a class="reference external" href="https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage">Cloud Storage connector for Apache Hadoop</a></p></li>
-</ul>
-<p>Create a new Dockerfile like the one shown below.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-images-recipes/hadoop.Dockerfile</span></p>
-<div class="highlight-dockerfile notranslate"><div class="highlight"><pre><span></span><span class="c"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
-<span class="c"># contributor license agreements. See the NOTICE file distributed with</span>
-<span class="c"># this work for additional information regarding copyright ownership.</span>
-<span class="c"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
-<span class="c"># (the "License"); you may not use this file except in compliance with</span>
-<span class="c"># the License. You may obtain a copy of the License at</span>
-<span class="c">#</span>
-<span class="c"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c">#</span>
-<span class="c"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c"># See the License for the specific language governing permissions and</span>
-<span class="c"># limitations under the License.</span>
-<span class="k">ARG</span> BASE_AIRFLOW_IMAGE
-
-<span class="k">FROM</span> <span class="s">${BASE_AIRFLOW_IMAGE}</span>
-
-<span class="k">SHELL</span> <span class="p">[</span><span class="s2">"/bin/bash"</span><span class="p">,</span> <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"pipefail"</span><span class="p">,</span> <span class="s2">"-e"</span><span class="p">,</span> <span class="s2">"-u"</span><span class="p">,</span> <span class="s2">"-x"</span><span class="p">,</span> <span class="s2">"-c"</span><span clas [...]
-
-<span class="k">USER</span><span class="s"> 0</span>
-
-<span class="c"># Install Java</span>
-<span class="k">RUN</span> mkdir -pv /usr/share/man/man1 <span class="se">\</span>
- <span class="o">&&</span> mkdir -pv /usr/share/man/man7 <span class="se">\</span>
- <span class="o">&&</span> curl -fsSL https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public <span class="p">|</span> apt-key add - <span class="se">\</span>
- <span class="o">&&</span> <span class="nb">echo</span> <span class="s1">'deb https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ buster main'</span> > <span class="se">\</span>
- /etc/apt/sources.list.d/adoptopenjdk.list <span class="se">\</span>
- <span class="o">&&</span> apt-get update <span class="se">\</span>
- <span class="o">&&</span> apt-get install --no-install-recommends -y <span class="se">\</span>
- adoptopenjdk-8-hotspot-jre <span class="se">\</span>
- <span class="o">&&</span> apt-get autoremove -yqq --purge <span class="se">\</span>
- <span class="o">&&</span> apt-get clean <span class="se">\</span>
- <span class="o">&&</span> rm -rf /var/lib/apt/lists/*
-<span class="k">ENV</span> <span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/adoptopenjdk-8-hotspot-jre-amd64
-
-<span class="k">RUN</span> mkdir -p /opt/spark/jars
-
-<span class="c"># Install Apache Hadoop</span>
-<span class="k">ARG</span> <span class="nv">HADOOP_VERSION</span><span class="o">=</span><span class="m">2</span>.10.1
-<span class="k">ENV</span> <span class="nv">HADOOP_HOME</span><span class="o">=</span>/opt/hadoop
-<span class="k">ENV</span> <span class="nv">HADOOP_CONF_DIR</span><span class="o">=</span>/etc/hadoop
-<span class="k">ENV</span> <span class="nv">MULTIHOMED_NETWORK</span><span class="o">=</span><span class="m">1</span>
-<span class="k">ENV</span> <span class="nv">USER</span><span class="o">=</span>root
-
-<span class="k">RUN</span> <span class="nv">HADOOP_URL</span><span class="o">=</span><span class="s2">"https://archive.apache.org/dist/hadoop/common/hadoop-</span><span class="nv">$HADOOP_VERSION</span><span class="s2">/hadoop-</span><span class="nv">$HADOOP_VERSION</span><span class="s2">.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> curl <span class="s1">'https://dist.apache.org/repos/dist/release/hadoop/common/KEYS'</span> <span class="p">|</span> gpg --import - <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HADOOP_URL</span><span class="s2">"</span> -o /tmp/hadoop.tar.gz <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HADOOP_URL</span><span class="s2">.asc"</span> -o /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> gpg --verify /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> tar -xvf /tmp/hadoop.tar.gz -C <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> rm /tmp/hadoop.tar.gz /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> ln -s <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">/etc/hadoop"</span> /etc/hadoop <span class="se">\</span>
- <span class="o">&&</span> mkdir <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">/logs"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir /hadoop-data
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="nv">$HADOOP_HOME</span><span class="s2">/bin/:</span><span class="nv">$PATH</span><span class="s2">"</span>
-
-<span class="c"># Install Apache Hive</span>
-<span class="k">ARG</span> <span class="nv">HIVE_VERSION</span><span class="o">=</span><span class="m">2</span>.3.7
-<span class="k">ENV</span> <span class="nv">HIVE_HOME</span><span class="o">=</span>/opt/hive
-<span class="k">ENV</span> <span class="nv">HIVE_CONF_DIR</span><span class="o">=</span>/etc/hive
-
-<span class="k">RUN</span> <span class="nv">HIVE_URL</span><span class="o">=</span><span class="s2">"https://archive.apache.org/dist/hive/hive-</span><span class="si">${</span><span class="nv">HIVE_VERSION</span><span class="si">}</span><span class="s2">/apache-hive-</span><span class="si">${</span><span class="nv">HIVE_VERSION</span><span class="si">}</span><span class="s2">-bin.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s1">'https://downloads.apache.org/hive/KEYS'</span> <span class="p">|</span> gpg --import - <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HIVE_URL</span><span class="s2">"</span> -o /tmp/hive.tar.gz <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HIVE_URL</span><span class="s2">.asc"</span> -o /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> gpg --verify /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> mkdir -p <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> tar -xf /tmp/hive.tar.gz -C <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> rm /tmp/hive.tar.gz /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> ln -s <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">/etc/hive"</span> <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_CONF_DIR</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">/logs"</span>
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="nv">$HIVE_HOME</span><span class="s2">/bin/:</span><span class="nv">$PATH</span><span class="s2">"</span>
-
-<span class="c"># Install GCS connector for Apache Hadoop</span>
-<span class="c"># See: https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage</span>
-<span class="k">ARG</span> <span class="nv">GCS_VARIANT</span><span class="o">=</span><span class="s2">"hadoop2"</span>
-<span class="k">ARG</span> <span class="nv">GCS_VERSION</span><span class="o">=</span><span class="s2">"2.1.5"</span>
-
-<span class="k">RUN</span> <span class="nv">GCS_JAR_PATH</span><span class="o">=</span><span class="s2">"/opt/spark/jars/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar"</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="nv">GCS_JAR_URL</span><span class="o">=</span><span class="s2">"https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar"</span> <span class="se">\</span>
- <span class="o">&&</span> curl <span class="s2">"</span><span class="si">${</span><span class="nv">GCS_JAR_URL</span><span class="si">}</span><span class="s2">"</span> -o <span class="s2">"</span><span class="si">${</span><span class="nv">GCS_JAR_PATH</span><span class="si">}</span><span class="s2">"</span>
-
-<span class="k">ENV</span> <span class="nv">HADOOP_CLASSPATH</span><span class="o">=</span><span class="s2">"/opt/spark/jars/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar:</span><span class="nv">$HADOOP_CLASSPATH</span><span class="s2">"</span>
-
-<span class="k">USER</span><span class="s"> ${AIRFLOW_UID}</span>
-</pre></div>
-</div>
-</div>
-<p>Then build a new image.</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">BASE_AIRFLOW_IMAGE</span><span class="o">=</span><span class="s2">"apache/airflow:2.0.2"</span> <span class="se">\</span>
- -t my-airflow-image
-</pre></div>
-</div>
+ <div class="section" id="module-airflow.executors.celery_kubernetes_executor">
+<span id="airflow-executors-celery-kubernetes-executor"></span><h1><a class="reference internal" href="#module-airflow.executors.celery_kubernetes_executor" title="airflow.executors.celery_kubernetes_executor"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a><a class="headerlink" href="#module-airflow.executors.celery_kubernetes_executor" title="Permalink to this headline">¶</a></h1>
+<div class="section" id="module-contents">
+<h2>Module Contents<a class="headerlink" href="#module-contents" title="Permalink to this headline">¶</a></h2>
+<dl class="py class">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor">
+<em class="property">class </em><code class="sig-prename descclassname">airflow.executors.celery_kubernetes_executor.</code><code class="sig-name descname">CeleryKubernetesExecutor</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">celery_executor</span></em>, <em class="sig-param"><span class="n">kubernetes_executor</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.htm [...]
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.utils.log.logging_mixin.LoggingMixin</span></code></p>
+<p>CeleryKubernetesExecutor consists of CeleryExecutor and KubernetesExecutor.
+It chooses an executor to use based on the queue defined on the task.
+When the queue is the value of <code class="docutils literal notranslate"><span class="pre">kubernetes_queue</span></code> in section <code class="docutils literal notranslate"><span class="pre">[celery_kubernetes_executor]</span></code>
+of the configuration (default value: <cite>kubernetes</cite>), KubernetesExecutor is selected to run the task,
+otherwise, CeleryExecutor is used.</p>
+<dl class="py attribute">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.KUBERNETES_QUEUE">
+<code class="sig-name descname">KUBERNETES_QUEUE</code><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.html#CeleryKubernetesExecutor.KUBERNETES_QUEUE"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.KUBERNETES_QUEUE" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py attribute">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.queued_tasks">
+<code class="sig-name descname">queued_tasks</code><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.html#CeleryKubernetesExecutor.queued_tasks"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.queued_tasks" title="Permalink to this definition">¶</a></dt>
+<dd><p>Return queued tasks from celery and kubernetes executor</p>
+</dd></dl>
+
+<dl class="py attribute">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.running">
+<code class="sig-name descname">running</code><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.html#CeleryKubernetesExecutor.running"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.running" title="Permalink to this definition">¶</a></dt>
+<dd><p>Return running tasks from celery and kubernetes executor</p>
+</dd></dl>
+
+<dl class="py attribute">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.job_id">
+<code class="sig-name descname">job_id</code><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.html#CeleryKubernetesExecutor.job_id"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.job_id" title="Permalink to this definition">¶</a></dt>
+<dd><p>This is a class attribute in BaseExecutor but since this is not really an executor, but a wrapper
+of executors we implement as property so we can have custom setter.</p>
+</dd></dl>
+
+<dl class="py attribute">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.slots_available">
+<code class="sig-name descname">slots_available</code><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.html#CeleryKubernetesExecutor.slots_available"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.slots_available" title="Permalink to this definition">¶</a></dt>
+<dd><p>Number of new tasks this executor instance can accept</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.start">
+<code class="sig-name descname">start</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.html#CeleryKubernetesExecutor.start"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.start" title="Permalink to this definition">¶ [...]
+<dd><p>Start celery and kubernetes executor</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.queue_command">
+<code class="sig-name descname">queue_command</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">task_instance</span><span class="p">:</span> <span class="n">TaskInstance</span></em>, <em class="sig-param"><span class="n">command</span><span class="p">:</span> <span class="n">CommandType</span></em>, <em class="sig-param"><span class="n">priority</span><span class="p">:</span> <span class="n"><a class="refer [...]
+<dd><p>Queues command via celery or kubernetes executor</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.queue_task_instance">
+<code class="sig-name descname">queue_task_instance</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">task_instance</span><span class="p">:</span> <span class="n">TaskInstance</span></em>, <em class="sig-param"><span class="n">mark_success</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.9)">bool</a [...]
+<dd><p>Queues task instance via celery or kubernetes executor</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.has_task">
+<code class="sig-name descname">has_task</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">task_instance</span><span class="p">:</span> <span class="n">TaskInstance</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.html#CeleryKubernetesExecutor.has_task"><span class="viewcode-link">[source]</span></a><a class="hea [...]
+<dd><p>Checks if a task is either queued or running in either celery or kubernetes executor.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>task_instance</strong> -- TaskInstance</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>True if the task is known to this executor</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.heartbeat">
+<code class="sig-name descname">heartbeat</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.html#CeleryKubernetesExecutor.heartbeat"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.heartbeat" title="Permalink to this d [...]
+<dd><p>Heartbeat sent to trigger new jobs in celery and kubernetes executor</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.get_event_buffer">
+<code class="sig-name descname">get_event_buffer</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">dag_ids</span><span class="o">=</span><span class="default_value">None</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.html#CeleryKubernetesExecutor.get_event_buffer"><span class="viewcode-link">[source]</span></a> [...]
+<dd><p>Returns and flush the event buffer from celery and kubernetes executor</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>dag_ids</strong> -- to dag_ids to return events for, if None returns all</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>a dict of events</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.try_adopt_task_instances">
+<code class="sig-name descname">try_adopt_task_instances</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">tis</span><span class="p">:</span> <span class="n">List<span class="p">[</span>TaskInstance<span class="p">]</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.html#CeleryKubernetesExecutor.try_adopt_ta [...]
+<dd><p>Try to adopt running task instances that have been abandoned by a SchedulerJob dying.</p>
+<p>Anything that is not adopted will be cleared by the scheduler (and then become eligible for
+re-scheduling)</p>
+<dl class="field-list simple">
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>any TaskInstances that were unable to be adopted</p>
+</dd>
+<dt class="field-even">Return type</dt>
+<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.9)">list</a>[<a class="reference internal" href="../../models/index.html#airflow.models.TaskInstance" title="airflow.models.TaskInstance">airflow.models.TaskInstance</a>]</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.end">
+<code class="sig-name descname">end</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.html#CeleryKubernetesExecutor.end"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.end" title="Permalink to this definition">¶</a></dt>
+<dd><p>End celery and kubernetes executor</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.terminate">
+<code class="sig-name descname">terminate</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.html#CeleryKubernetesExecutor.terminate"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.terminate" title="Permalink to this d [...]
+<dd><p>Terminate celery and kubernetes executor</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor._router">
+<code class="sig-name descname">_router</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">simple_task_instance</span><span class="p">:</span> <span class="n">SimpleTaskInstance</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.html#CeleryKubernetesExecutor._router"><span class="viewcode-link">[source]</span></a><a [...]
+<dd><p>Return either celery_executor or kubernetes_executor</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>simple_task_instance</strong> -- SimpleTaskInstance</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>celery_executor or kubernetes_executor</p>
+</dd>
+<dt class="field-odd">Return type</dt>
+<dd class="field-odd"><p>Union[<a class="reference internal" href="../celery_executor/index.html#airflow.executors.celery_executor.CeleryExecutor" title="airflow.executors.celery_executor.CeleryExecutor">CeleryExecutor</a>, <a class="reference internal" href="../kubernetes_executor/index.html#airflow.executors.kubernetes_executor.KubernetesExecutor" title="airflow.executors.kubernetes_executor.KubernetesExecutor">KubernetesExecutor</a>]</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.debug_dump">
+<code class="sig-name descname">debug_dump</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/celery_kubernetes_executor.html#CeleryKubernetesExecutor.debug_dump"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.debug_dump" title="Permalink to thi [...]
+<dd><p>Called in response to SIGUSR2 by the scheduler</p>
+</dd></dl>
+
+</dd></dl>
+
</div>
</div>
<div class="pager" role="navigation" aria-label="related navigation">
- <a rel="prev" title="Entrypoint" href="entrypoint.html" >
+ <a rel="prev" title="airflow.executors.celery_executor" href="../celery_executor/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="p">Previous</button>
</a>
- <a rel="next" title="Image build arguments reference" href="build-arg-ref.html" >
+ <a rel="next" title="airflow.executors.dask_executor" href="../dask_executor/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="n">Next</button>
</a>
</div>
@@ -819,9 +1023,59 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
<nav class="wy-nav-side-toc">
<div class="wy-menu-vertical">
<ul>
-<li><a class="reference internal" href="#">Recipes</a><ul>
-<li><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
+<li><a class="reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a><ul>
+<li><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor">CeleryKubernetesExecutor</a><ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.KUBERNETES_QUEUE">KUBERNETES_QUEUE</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.queued_tasks">queued_tasks</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.running">running</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.job_id">job_id</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.slots_available">slots_available</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.start">start</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.queue_command">queue_command</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.queue_task_instance">queue_task_instance</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.has_task">has_task</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.heartbeat">heartbeat</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.get_event_buffer">get_event_buffer</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.try_adopt_task_instances">try_adopt_task_instances</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.end">end</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.terminate">terminate</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor._router">_router</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.celery_kubernetes_executor.CeleryKubernetesExecutor.debug_dump">debug_dump</a></li>
+</ul>
+</li>
+</ul>
+</li>
</ul>
</li>
</ul>
@@ -846,7 +1100,7 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
<div class="base-layout--button">
- <a href="https://github.com/apache/airflow/edit/devel/docs/docker-stack/recipes.rst" rel="nofollow">
+ <a href="https://github.com/apache/airflow/edit/devel/docs/apache-airflow/_api/airflow/executors/celery_kubernetes_executor/index.rst" rel="nofollow">
<button class="btn-hollow btn-brown btn-with-icon with-box-shadow button-fixed">
<svg xmlns="http://www.w3.org/2000/svg" width="30.76" height="30">
@@ -980,15 +1234,16 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
</span>
</div>
</footer>
- <script type="text/javascript" src="_static/_gen/js/docs.js"></script>
- <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/_gen/js/docs.js"></script>
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
<script src="https://code.jquery.com/jquery-3.4.1.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script>
- <script src="_static/jquery.js"></script>
- <script src="_static/underscore.js"></script>
- <script src="_static/doctools.js"></script>
- <script src="_static/clipboard.min.js"></script>
- <script src="_static/copybutton.js"></script>
+ <script src="../../../../_static/jquery.js"></script>
+ <script src="../../../../_static/underscore.js"></script>
+ <script src="../../../../_static/doctools.js"></script>
+ <script src="../../../../_static/clipboard.min.js"></script>
+ <script src="../../../../_static/copybutton.js"></script>
+ <script src="../../../../_static/jira-links.js"></script>
</body>
</html>
\ No newline at end of file
diff --git a/docs-archive/docker-stack/recipes.html b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/dask_executor/index.html
similarity index 73%
copy from docs-archive/docker-stack/recipes.html
copy to docs-archive/apache-airflow/2.1.0/_api/airflow/executors/dask_executor/index.html
index 75019af..ef59e6a 100644
--- a/docs-archive/docker-stack/recipes.html
+++ b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/dask_executor/index.html
@@ -9,15 +9,16 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Recipes — docker-stack Documentation</title>
- <link rel="stylesheet" href="_static/_gen/css/main.min.css" type="text/css" />
- <link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
- <link rel="stylesheet" type="text/css" href="_static/_gen/css/main-custom.min.css" />
- <link rel="shortcut icon" href="_static/pin_32.png"/>
- <link rel="index" title="Index" href="genindex.html" />
- <link rel="search" title="Search" href="search.html" />
- <link rel="next" title="Image build arguments reference" href="build-arg-ref.html" />
- <link rel="prev" title="Entrypoint" href="entrypoint.html" />
+ <title>airflow.executors.dask_executor — Airflow Documentation</title>
+ <link rel="stylesheet" href="../../../../_static/_gen/css/main.min.css" type="text/css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/copybutton.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/_gen/css/main-custom.min.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/graphviz.css" />
+ <link rel="shortcut icon" href="../../../../_static/pin_32.png"/>
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+ <link rel="next" title="airflow.executors.debug_executor" href="../debug_executor/index.html" />
+ <link rel="prev" title="airflow.executors.celery_kubernetes_executor" href="../celery_kubernetes_executor/index.html" />
<script type="application/javascript">
var doNotTrack = false;
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
@@ -272,7 +273,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -283,7 +284,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -313,19 +314,103 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Recipes</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
-</ul>
-</li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6"><a class="reference internal" href="../base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor">DaskExecutor</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.start">start</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.execute_async">execute_async</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor._process_future">_process_future</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.sync">sync</a></li>
+</ul>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.end">end</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.terminate">terminate</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l6"><a class="reference internal" href="../debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -409,7 +494,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -420,7 +505,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -450,19 +535,103 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Recipes</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
-</ul>
-</li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6"><a class="reference internal" href="../base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor">DaskExecutor</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.start">start</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.execute_async">execute_async</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor._process_future">_process_future</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.sync">sync</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.end">end</a></li>
+</ul>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.terminate">terminate</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l6"><a class="reference internal" href="../debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -551,9 +720,13 @@
<ul class="breadcrumb">
- <li class="breadcrumb-item"><a href="index.html" class="icon icon-home"> Home</a></li>
+ <li class="breadcrumb-item"><a href="../../../../index.html" class="icon icon-home"> Home</a></li>
+
+ <li class="breadcrumb-item"><a href="../../../../python-api-ref.html">Python API Reference</a></li>
+
+ <li class="breadcrumb-item"><a href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a></li>
- <li class="breadcrumb-item"><a href="recipes.html"> Recipes</a></li>
+ <li class="breadcrumb-item"><a href="index.html"> <code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
</ul>
</div>
@@ -564,192 +737,63 @@
<div class="bodywrapper">
<div class="body" role="main">
- <blockquote>
-<div></div></blockquote>
-<div class="section" id="recipes">
-<h1>Recipes<a class="headerlink" href="#recipes" title="Permalink to this headline">¶</a></h1>
-<p>Users sometimes share interesting ways of using the Docker images. We encourage users to contribute these
-recipes to the documentation in case they prove useful to other members of the community by
-submitting a pull request. The sections below capture this knowledge.</p>
-<div class="section" id="google-cloud-sdk-installation">
-<h2>Google Cloud SDK installation<a class="headerlink" href="#google-cloud-sdk-installation" title="Permalink to this headline">¶</a></h2>
-<p>Some operators, such as <a class="reference external" href="/docs/apache-airflow-providers-google/stable/_api/airflow/providers/google/cloud/operators/kubernetes_engine/index.html#airflow.providers.google.cloud.operators.kubernetes_engine.GKEStartPodOperator" title="(in apache-airflow-providers-google vdevel)"><code class="xref py py-class docutils literal notranslate"><span class="pre">GKEStartPodOperator</span></code></a>,
-<a class="reference external" href="/docs/apache-airflow-providers-google/stable/_api/airflow/providers/google/cloud/operators/dataflow/index.html#airflow.providers.google.cloud.operators.dataflow.DataflowStartSqlJobOperator" title="(in apache-airflow-providers-google vdevel)"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataflowStartSqlJobOperator</span></code></a>, require
-the installation of <a class="reference external" href="https://cloud.google.com/sdk">Google Cloud SDK</a> (includes <code class="docutils literal notranslate"><span class="pre">gcloud</span></code>).
-You can also run these commands with BashOperator.</p>
-<p>Create a new Dockerfile like the one shown below.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-images-recipes/gcloud.Dockerfile</span></p>
-<div class="highlight-dockerfile notranslate"><div class="highlight"><pre><span></span><span class="c"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
-<span class="c"># contributor license agreements. See the NOTICE file distributed with</span>
-<span class="c"># this work for additional information regarding copyright ownership.</span>
-<span class="c"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
-<span class="c"># (the "License"); you may not use this file except in compliance with</span>
-<span class="c"># the License. You may obtain a copy of the License at</span>
-<span class="c">#</span>
-<span class="c"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c">#</span>
-<span class="c"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c"># See the License for the specific language governing permissions and</span>
-<span class="c"># limitations under the License.</span>
-<span class="k">ARG</span> BASE_AIRFLOW_IMAGE
-<span class="k">FROM</span> <span class="s">${BASE_AIRFLOW_IMAGE}</span>
-
-<span class="k">SHELL</span> <span class="p">[</span><span class="s2">"/bin/bash"</span><span class="p">,</span> <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"pipefail"</span><span class="p">,</span> <span class="s2">"-e"</span><span class="p">,</span> <span class="s2">"-u"</span><span class="p">,</span> <span class="s2">"-x"</span><span class="p">,</span> <span class="s2">"-c"</span><span clas [...]
-
-<span class="k">USER</span><span class="s"> 0</span>
-
-<span class="k">ARG</span> <span class="nv">CLOUD_SDK_VERSION</span><span class="o">=</span><span class="m">322</span>.0.0
-<span class="k">ENV</span> <span class="nv">GCLOUD_HOME</span><span class="o">=</span>/opt/google-cloud-sdk
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">/bin/:</span><span class="si">${</span><span class="nv">PATH</span><span class="si">}</span><span class="s2">"</span>
-
-<span class="k">RUN</span> <span class="nv">DOWNLOAD_URL</span><span class="o">=</span><span class="s2">"https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-</span><span class="si">${</span><span class="nv">CLOUD_SDK_VERSION</span><span class="si">}</span><span class="s2">-linux-x86_64.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="nv">TMP_DIR</span><span class="o">=</span><span class="s2">"</span><span class="k">$(</span>mktemp -d<span class="k">)</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> curl -fL <span class="s2">"</span><span class="si">${</span><span class="nv">DOWNLOAD_URL</span><span class="si">}</span><span class="s2">"</span> --output <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">/google-cloud-sdk.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir -p <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> tar xzf <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">/google-cloud-sdk.tar.gz"</span> -C <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">/install.sh"</span> <span class="se">\</span>
- --bash-completion<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --path-update<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --usage-reporting<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --additional-components alpha beta kubectl <span class="se">\</span>
- --quiet <span class="se">\</span>
- <span class="o">&&</span> rm -rf <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> gcloud --version
-
-<span class="k">USER</span><span class="s"> ${AIRFLOW_UID}</span>
-</pre></div>
-</div>
-</div>
-<p>Then build a new image.</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">BASE_AIRFLOW_IMAGE</span><span class="o">=</span><span class="s2">"apache/airflow:2.0.2"</span> <span class="se">\</span>
- -t my-airflow-image
-</pre></div>
-</div>
-</div>
-<div class="section" id="apache-hadoop-stack-installation">
-<h2>Apache Hadoop Stack installation<a class="headerlink" href="#apache-hadoop-stack-installation" title="Permalink to this headline">¶</a></h2>
-<p>Airflow is often used to run tasks on Hadoop cluster. It required Java Runtime Environment (JRE) to run.
-Below are the steps to take tools that are frequently used in Hadoop-world:</p>
-<ul class="simple">
-<li><p>Java Runtime Environment (JRE)</p></li>
-<li><p>Apache Hadoop</p></li>
-<li><p>Apache Hive</p></li>
-<li><p><a class="reference external" href="https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage">Cloud Storage connector for Apache Hadoop</a></p></li>
-</ul>
-<p>Create a new Dockerfile like the one shown below.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-images-recipes/hadoop.Dockerfile</span></p>
-<div class="highlight-dockerfile notranslate"><div class="highlight"><pre><span></span><span class="c"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
-<span class="c"># contributor license agreements. See the NOTICE file distributed with</span>
-<span class="c"># this work for additional information regarding copyright ownership.</span>
-<span class="c"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
-<span class="c"># (the "License"); you may not use this file except in compliance with</span>
-<span class="c"># the License. You may obtain a copy of the License at</span>
-<span class="c">#</span>
-<span class="c"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c">#</span>
-<span class="c"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c"># See the License for the specific language governing permissions and</span>
-<span class="c"># limitations under the License.</span>
-<span class="k">ARG</span> BASE_AIRFLOW_IMAGE
-
-<span class="k">FROM</span> <span class="s">${BASE_AIRFLOW_IMAGE}</span>
-
-<span class="k">SHELL</span> <span class="p">[</span><span class="s2">"/bin/bash"</span><span class="p">,</span> <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"pipefail"</span><span class="p">,</span> <span class="s2">"-e"</span><span class="p">,</span> <span class="s2">"-u"</span><span class="p">,</span> <span class="s2">"-x"</span><span class="p">,</span> <span class="s2">"-c"</span><span clas [...]
-
-<span class="k">USER</span><span class="s"> 0</span>
-
-<span class="c"># Install Java</span>
-<span class="k">RUN</span> mkdir -pv /usr/share/man/man1 <span class="se">\</span>
- <span class="o">&&</span> mkdir -pv /usr/share/man/man7 <span class="se">\</span>
- <span class="o">&&</span> curl -fsSL https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public <span class="p">|</span> apt-key add - <span class="se">\</span>
- <span class="o">&&</span> <span class="nb">echo</span> <span class="s1">'deb https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ buster main'</span> > <span class="se">\</span>
- /etc/apt/sources.list.d/adoptopenjdk.list <span class="se">\</span>
- <span class="o">&&</span> apt-get update <span class="se">\</span>
- <span class="o">&&</span> apt-get install --no-install-recommends -y <span class="se">\</span>
- adoptopenjdk-8-hotspot-jre <span class="se">\</span>
- <span class="o">&&</span> apt-get autoremove -yqq --purge <span class="se">\</span>
- <span class="o">&&</span> apt-get clean <span class="se">\</span>
- <span class="o">&&</span> rm -rf /var/lib/apt/lists/*
-<span class="k">ENV</span> <span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/adoptopenjdk-8-hotspot-jre-amd64
-
-<span class="k">RUN</span> mkdir -p /opt/spark/jars
-
-<span class="c"># Install Apache Hadoop</span>
-<span class="k">ARG</span> <span class="nv">HADOOP_VERSION</span><span class="o">=</span><span class="m">2</span>.10.1
-<span class="k">ENV</span> <span class="nv">HADOOP_HOME</span><span class="o">=</span>/opt/hadoop
-<span class="k">ENV</span> <span class="nv">HADOOP_CONF_DIR</span><span class="o">=</span>/etc/hadoop
-<span class="k">ENV</span> <span class="nv">MULTIHOMED_NETWORK</span><span class="o">=</span><span class="m">1</span>
-<span class="k">ENV</span> <span class="nv">USER</span><span class="o">=</span>root
-
-<span class="k">RUN</span> <span class="nv">HADOOP_URL</span><span class="o">=</span><span class="s2">"https://archive.apache.org/dist/hadoop/common/hadoop-</span><span class="nv">$HADOOP_VERSION</span><span class="s2">/hadoop-</span><span class="nv">$HADOOP_VERSION</span><span class="s2">.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> curl <span class="s1">'https://dist.apache.org/repos/dist/release/hadoop/common/KEYS'</span> <span class="p">|</span> gpg --import - <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HADOOP_URL</span><span class="s2">"</span> -o /tmp/hadoop.tar.gz <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HADOOP_URL</span><span class="s2">.asc"</span> -o /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> gpg --verify /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> tar -xvf /tmp/hadoop.tar.gz -C <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> rm /tmp/hadoop.tar.gz /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> ln -s <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">/etc/hadoop"</span> /etc/hadoop <span class="se">\</span>
- <span class="o">&&</span> mkdir <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">/logs"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir /hadoop-data
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="nv">$HADOOP_HOME</span><span class="s2">/bin/:</span><span class="nv">$PATH</span><span class="s2">"</span>
-
-<span class="c"># Install Apache Hive</span>
-<span class="k">ARG</span> <span class="nv">HIVE_VERSION</span><span class="o">=</span><span class="m">2</span>.3.7
-<span class="k">ENV</span> <span class="nv">HIVE_HOME</span><span class="o">=</span>/opt/hive
-<span class="k">ENV</span> <span class="nv">HIVE_CONF_DIR</span><span class="o">=</span>/etc/hive
-
-<span class="k">RUN</span> <span class="nv">HIVE_URL</span><span class="o">=</span><span class="s2">"https://archive.apache.org/dist/hive/hive-</span><span class="si">${</span><span class="nv">HIVE_VERSION</span><span class="si">}</span><span class="s2">/apache-hive-</span><span class="si">${</span><span class="nv">HIVE_VERSION</span><span class="si">}</span><span class="s2">-bin.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s1">'https://downloads.apache.org/hive/KEYS'</span> <span class="p">|</span> gpg --import - <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HIVE_URL</span><span class="s2">"</span> -o /tmp/hive.tar.gz <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HIVE_URL</span><span class="s2">.asc"</span> -o /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> gpg --verify /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> mkdir -p <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> tar -xf /tmp/hive.tar.gz -C <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> rm /tmp/hive.tar.gz /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> ln -s <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">/etc/hive"</span> <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_CONF_DIR</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">/logs"</span>
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="nv">$HIVE_HOME</span><span class="s2">/bin/:</span><span class="nv">$PATH</span><span class="s2">"</span>
-
-<span class="c"># Install GCS connector for Apache Hadoop</span>
-<span class="c"># See: https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage</span>
-<span class="k">ARG</span> <span class="nv">GCS_VARIANT</span><span class="o">=</span><span class="s2">"hadoop2"</span>
-<span class="k">ARG</span> <span class="nv">GCS_VERSION</span><span class="o">=</span><span class="s2">"2.1.5"</span>
-
-<span class="k">RUN</span> <span class="nv">GCS_JAR_PATH</span><span class="o">=</span><span class="s2">"/opt/spark/jars/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar"</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="nv">GCS_JAR_URL</span><span class="o">=</span><span class="s2">"https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar"</span> <span class="se">\</span>
- <span class="o">&&</span> curl <span class="s2">"</span><span class="si">${</span><span class="nv">GCS_JAR_URL</span><span class="si">}</span><span class="s2">"</span> -o <span class="s2">"</span><span class="si">${</span><span class="nv">GCS_JAR_PATH</span><span class="si">}</span><span class="s2">"</span>
-
-<span class="k">ENV</span> <span class="nv">HADOOP_CLASSPATH</span><span class="o">=</span><span class="s2">"/opt/spark/jars/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar:</span><span class="nv">$HADOOP_CLASSPATH</span><span class="s2">"</span>
-
-<span class="k">USER</span><span class="s"> ${AIRFLOW_UID}</span>
-</pre></div>
-</div>
-</div>
-<p>Then build a new image.</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">BASE_AIRFLOW_IMAGE</span><span class="o">=</span><span class="s2">"apache/airflow:2.0.2"</span> <span class="se">\</span>
- -t my-airflow-image
-</pre></div>
+ <div class="section" id="module-airflow.executors.dask_executor">
+<span id="airflow-executors-dask-executor"></span><h1><a class="reference internal" href="#module-airflow.executors.dask_executor" title="airflow.executors.dask_executor"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a><a class="headerlink" href="#module-airflow.executors.dask_executor" title="Permalink to this headline">¶</a></h1>
+<p><p>DaskExecutor</p>
+<div class="admonition seealso">
+<p class="admonition-title">See also</p>
+<p>For more information on how the DaskExecutor works, take a look at the guide:
+<a class="reference internal" href="../../../../executor/dask.html#executor-daskexecutor"><span class="std std-ref">Dask Executor</span></a></p>
</div>
+</p>
+<div class="section" id="module-contents">
+<h2>Module Contents<a class="headerlink" href="#module-contents" title="Permalink to this headline">¶</a></h2>
+<dl class="py class">
+<dt id="airflow.executors.dask_executor.DaskExecutor">
+<em class="property">class </em><code class="sig-prename descclassname">airflow.executors.dask_executor.</code><code class="sig-name descname">DaskExecutor</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">cluster_address</span><span class="o">=</span><span class="default_value">None</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/dask_executor.html#DaskExecutor"><span class="viewcode-link">[s [...]
+<dd><p>Bases: <a class="reference internal" href="../base_executor/index.html#airflow.executors.base_executor.BaseExecutor" title="airflow.executors.base_executor.BaseExecutor"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.executors.base_executor.BaseExecutor</span></code></a></p>
+<p>DaskExecutor submits tasks to a Dask Distributed cluster.</p>
+<dl class="py method">
+<dt id="airflow.executors.dask_executor.DaskExecutor.start">
+<code class="sig-name descname">start</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/dask_executor.html#DaskExecutor.start"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.dask_executor.DaskExecutor.start" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.dask_executor.DaskExecutor.execute_async">
+<code class="sig-name descname">execute_async</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">key</span><span class="p">:</span> <span class="n">TaskInstanceKey</span></em>, <em class="sig-param"><span class="n">command</span><span class="p">:</span> <span class="n">CommandType</span></em>, <em class="sig-param"><span class="n">queue</span><span class="p">:</span> <span class="n">Optional<span class="p">[ [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.dask_executor.DaskExecutor._process_future">
+<code class="sig-name descname">_process_future</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">future</span><span class="p">:</span> <span class="n">Future</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/dask_executor.html#DaskExecutor._process_future"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow. [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.dask_executor.DaskExecutor.sync">
+<code class="sig-name descname">sync</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/dask_executor.html#DaskExecutor.sync"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.dask_executor.DaskExecutor.sync" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.dask_executor.DaskExecutor.end">
+<code class="sig-name descname">end</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/dask_executor.html#DaskExecutor.end"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.dask_executor.DaskExecutor.end" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.dask_executor.DaskExecutor.terminate">
+<code class="sig-name descname">terminate</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/dask_executor.html#DaskExecutor.terminate"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.dask_executor.DaskExecutor.terminate" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+</dd></dl>
+
</div>
</div>
<div class="pager" role="navigation" aria-label="related navigation">
- <a rel="prev" title="Entrypoint" href="entrypoint.html" >
+ <a rel="prev" title="airflow.executors.celery_kubernetes_executor" href="../celery_kubernetes_executor/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="p">Previous</button>
</a>
- <a rel="next" title="Image build arguments reference" href="build-arg-ref.html" >
+ <a rel="next" title="airflow.executors.debug_executor" href="../debug_executor/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="n">Next</button>
</a>
</div>
@@ -819,9 +863,29 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
<nav class="wy-nav-side-toc">
<div class="wy-menu-vertical">
<ul>
-<li><a class="reference internal" href="#">Recipes</a><ul>
-<li><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
+<li><a class="reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a><ul>
+<li><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor">DaskExecutor</a><ul>
+<li><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.start">start</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.execute_async">execute_async</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor._process_future">_process_future</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.sync">sync</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.end">end</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.dask_executor.DaskExecutor.terminate">terminate</a></li>
+</ul>
+</li>
+</ul>
+</li>
</ul>
</li>
</ul>
@@ -846,7 +910,7 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
<div class="base-layout--button">
- <a href="https://github.com/apache/airflow/edit/devel/docs/docker-stack/recipes.rst" rel="nofollow">
+ <a href="https://github.com/apache/airflow/edit/devel/docs/apache-airflow/_api/airflow/executors/dask_executor/index.rst" rel="nofollow">
<button class="btn-hollow btn-brown btn-with-icon with-box-shadow button-fixed">
<svg xmlns="http://www.w3.org/2000/svg" width="30.76" height="30">
@@ -980,15 +1044,16 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
</span>
</div>
</footer>
- <script type="text/javascript" src="_static/_gen/js/docs.js"></script>
- <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/_gen/js/docs.js"></script>
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
<script src="https://code.jquery.com/jquery-3.4.1.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script>
- <script src="_static/jquery.js"></script>
- <script src="_static/underscore.js"></script>
- <script src="_static/doctools.js"></script>
- <script src="_static/clipboard.min.js"></script>
- <script src="_static/copybutton.js"></script>
+ <script src="../../../../_static/jquery.js"></script>
+ <script src="../../../../_static/underscore.js"></script>
+ <script src="../../../../_static/doctools.js"></script>
+ <script src="../../../../_static/clipboard.min.js"></script>
+ <script src="../../../../_static/copybutton.js"></script>
+ <script src="../../../../_static/jira-links.js"></script>
</body>
</html>
\ No newline at end of file
diff --git a/docs-archive/docker-stack/recipes.html b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/debug_executor/index.html
similarity index 69%
copy from docs-archive/docker-stack/recipes.html
copy to docs-archive/apache-airflow/2.1.0/_api/airflow/executors/debug_executor/index.html
index 75019af..d8af022 100644
--- a/docs-archive/docker-stack/recipes.html
+++ b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/debug_executor/index.html
@@ -9,15 +9,16 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Recipes — docker-stack Documentation</title>
- <link rel="stylesheet" href="_static/_gen/css/main.min.css" type="text/css" />
- <link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
- <link rel="stylesheet" type="text/css" href="_static/_gen/css/main-custom.min.css" />
- <link rel="shortcut icon" href="_static/pin_32.png"/>
- <link rel="index" title="Index" href="genindex.html" />
- <link rel="search" title="Search" href="search.html" />
- <link rel="next" title="Image build arguments reference" href="build-arg-ref.html" />
- <link rel="prev" title="Entrypoint" href="entrypoint.html" />
+ <title>airflow.executors.debug_executor — Airflow Documentation</title>
+ <link rel="stylesheet" href="../../../../_static/_gen/css/main.min.css" type="text/css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/copybutton.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/_gen/css/main-custom.min.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/graphviz.css" />
+ <link rel="shortcut icon" href="../../../../_static/pin_32.png"/>
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+ <link rel="next" title="airflow.executors.executor_constants" href="../executor_constants/index.html" />
+ <link rel="prev" title="airflow.executors.dask_executor" href="../dask_executor/index.html" />
<script type="application/javascript">
var doNotTrack = false;
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
@@ -272,7 +273,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -283,7 +284,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -313,19 +314,112 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Recipes</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
-</ul>
-</li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6"><a class="reference internal" href="../base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor">DebugExecutor</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor._terminated">_terminated</a></li>
+</ul>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.execute_async">execute_async</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.sync">sync</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor._run_task">_run_task</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.queue_task_instance">queue_task_instance</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.trigger_tasks">trigger_tasks</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.end">end</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.terminate">terminate</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.change_state">change_state</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -409,7 +503,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -420,7 +514,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -450,19 +544,112 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Recipes</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
-</ul>
-</li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6"><a class="reference internal" href="../base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor">DebugExecutor</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor._terminated">_terminated</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.execute_async">execute_async</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.sync">sync</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor._run_task">_run_task</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.queue_task_instance">queue_task_instance</a></li>
+</ul>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.trigger_tasks">trigger_tasks</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.end">end</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.terminate">terminate</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.change_state">change_state</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -551,9 +738,13 @@
<ul class="breadcrumb">
- <li class="breadcrumb-item"><a href="index.html" class="icon icon-home"> Home</a></li>
+ <li class="breadcrumb-item"><a href="../../../../index.html" class="icon icon-home"> Home</a></li>
+
+ <li class="breadcrumb-item"><a href="../../../../python-api-ref.html">Python API Reference</a></li>
+
+ <li class="breadcrumb-item"><a href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a></li>
- <li class="breadcrumb-item"><a href="recipes.html"> Recipes</a></li>
+ <li class="breadcrumb-item"><a href="index.html"> <code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
</ul>
</div>
@@ -564,192 +755,91 @@
<div class="bodywrapper">
<div class="body" role="main">
- <blockquote>
-<div></div></blockquote>
-<div class="section" id="recipes">
-<h1>Recipes<a class="headerlink" href="#recipes" title="Permalink to this headline">¶</a></h1>
-<p>Users sometimes share interesting ways of using the Docker images. We encourage users to contribute these
-recipes to the documentation in case they prove useful to other members of the community by
-submitting a pull request. The sections below capture this knowledge.</p>
-<div class="section" id="google-cloud-sdk-installation">
-<h2>Google Cloud SDK installation<a class="headerlink" href="#google-cloud-sdk-installation" title="Permalink to this headline">¶</a></h2>
-<p>Some operators, such as <a class="reference external" href="/docs/apache-airflow-providers-google/stable/_api/airflow/providers/google/cloud/operators/kubernetes_engine/index.html#airflow.providers.google.cloud.operators.kubernetes_engine.GKEStartPodOperator" title="(in apache-airflow-providers-google vdevel)"><code class="xref py py-class docutils literal notranslate"><span class="pre">GKEStartPodOperator</span></code></a>,
-<a class="reference external" href="/docs/apache-airflow-providers-google/stable/_api/airflow/providers/google/cloud/operators/dataflow/index.html#airflow.providers.google.cloud.operators.dataflow.DataflowStartSqlJobOperator" title="(in apache-airflow-providers-google vdevel)"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataflowStartSqlJobOperator</span></code></a>, require
-the installation of <a class="reference external" href="https://cloud.google.com/sdk">Google Cloud SDK</a> (includes <code class="docutils literal notranslate"><span class="pre">gcloud</span></code>).
-You can also run these commands with BashOperator.</p>
-<p>Create a new Dockerfile like the one shown below.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-images-recipes/gcloud.Dockerfile</span></p>
-<div class="highlight-dockerfile notranslate"><div class="highlight"><pre><span></span><span class="c"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
-<span class="c"># contributor license agreements. See the NOTICE file distributed with</span>
-<span class="c"># this work for additional information regarding copyright ownership.</span>
-<span class="c"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
-<span class="c"># (the "License"); you may not use this file except in compliance with</span>
-<span class="c"># the License. You may obtain a copy of the License at</span>
-<span class="c">#</span>
-<span class="c"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c">#</span>
-<span class="c"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c"># See the License for the specific language governing permissions and</span>
-<span class="c"># limitations under the License.</span>
-<span class="k">ARG</span> BASE_AIRFLOW_IMAGE
-<span class="k">FROM</span> <span class="s">${BASE_AIRFLOW_IMAGE}</span>
-
-<span class="k">SHELL</span> <span class="p">[</span><span class="s2">"/bin/bash"</span><span class="p">,</span> <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"pipefail"</span><span class="p">,</span> <span class="s2">"-e"</span><span class="p">,</span> <span class="s2">"-u"</span><span class="p">,</span> <span class="s2">"-x"</span><span class="p">,</span> <span class="s2">"-c"</span><span clas [...]
-
-<span class="k">USER</span><span class="s"> 0</span>
-
-<span class="k">ARG</span> <span class="nv">CLOUD_SDK_VERSION</span><span class="o">=</span><span class="m">322</span>.0.0
-<span class="k">ENV</span> <span class="nv">GCLOUD_HOME</span><span class="o">=</span>/opt/google-cloud-sdk
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">/bin/:</span><span class="si">${</span><span class="nv">PATH</span><span class="si">}</span><span class="s2">"</span>
-
-<span class="k">RUN</span> <span class="nv">DOWNLOAD_URL</span><span class="o">=</span><span class="s2">"https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-</span><span class="si">${</span><span class="nv">CLOUD_SDK_VERSION</span><span class="si">}</span><span class="s2">-linux-x86_64.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="nv">TMP_DIR</span><span class="o">=</span><span class="s2">"</span><span class="k">$(</span>mktemp -d<span class="k">)</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> curl -fL <span class="s2">"</span><span class="si">${</span><span class="nv">DOWNLOAD_URL</span><span class="si">}</span><span class="s2">"</span> --output <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">/google-cloud-sdk.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir -p <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> tar xzf <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">/google-cloud-sdk.tar.gz"</span> -C <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">/install.sh"</span> <span class="se">\</span>
- --bash-completion<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --path-update<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --usage-reporting<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --additional-components alpha beta kubectl <span class="se">\</span>
- --quiet <span class="se">\</span>
- <span class="o">&&</span> rm -rf <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> gcloud --version
-
-<span class="k">USER</span><span class="s"> ${AIRFLOW_UID}</span>
-</pre></div>
-</div>
-</div>
-<p>Then build a new image.</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">BASE_AIRFLOW_IMAGE</span><span class="o">=</span><span class="s2">"apache/airflow:2.0.2"</span> <span class="se">\</span>
- -t my-airflow-image
-</pre></div>
-</div>
-</div>
-<div class="section" id="apache-hadoop-stack-installation">
-<h2>Apache Hadoop Stack installation<a class="headerlink" href="#apache-hadoop-stack-installation" title="Permalink to this headline">¶</a></h2>
-<p>Airflow is often used to run tasks on Hadoop cluster. It required Java Runtime Environment (JRE) to run.
-Below are the steps to take tools that are frequently used in Hadoop-world:</p>
-<ul class="simple">
-<li><p>Java Runtime Environment (JRE)</p></li>
-<li><p>Apache Hadoop</p></li>
-<li><p>Apache Hive</p></li>
-<li><p><a class="reference external" href="https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage">Cloud Storage connector for Apache Hadoop</a></p></li>
-</ul>
-<p>Create a new Dockerfile like the one shown below.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-images-recipes/hadoop.Dockerfile</span></p>
-<div class="highlight-dockerfile notranslate"><div class="highlight"><pre><span></span><span class="c"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
-<span class="c"># contributor license agreements. See the NOTICE file distributed with</span>
-<span class="c"># this work for additional information regarding copyright ownership.</span>
-<span class="c"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
-<span class="c"># (the "License"); you may not use this file except in compliance with</span>
-<span class="c"># the License. You may obtain a copy of the License at</span>
-<span class="c">#</span>
-<span class="c"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c">#</span>
-<span class="c"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c"># See the License for the specific language governing permissions and</span>
-<span class="c"># limitations under the License.</span>
-<span class="k">ARG</span> BASE_AIRFLOW_IMAGE
-
-<span class="k">FROM</span> <span class="s">${BASE_AIRFLOW_IMAGE}</span>
-
-<span class="k">SHELL</span> <span class="p">[</span><span class="s2">"/bin/bash"</span><span class="p">,</span> <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"pipefail"</span><span class="p">,</span> <span class="s2">"-e"</span><span class="p">,</span> <span class="s2">"-u"</span><span class="p">,</span> <span class="s2">"-x"</span><span class="p">,</span> <span class="s2">"-c"</span><span clas [...]
-
-<span class="k">USER</span><span class="s"> 0</span>
-
-<span class="c"># Install Java</span>
-<span class="k">RUN</span> mkdir -pv /usr/share/man/man1 <span class="se">\</span>
- <span class="o">&&</span> mkdir -pv /usr/share/man/man7 <span class="se">\</span>
- <span class="o">&&</span> curl -fsSL https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public <span class="p">|</span> apt-key add - <span class="se">\</span>
- <span class="o">&&</span> <span class="nb">echo</span> <span class="s1">'deb https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ buster main'</span> > <span class="se">\</span>
- /etc/apt/sources.list.d/adoptopenjdk.list <span class="se">\</span>
- <span class="o">&&</span> apt-get update <span class="se">\</span>
- <span class="o">&&</span> apt-get install --no-install-recommends -y <span class="se">\</span>
- adoptopenjdk-8-hotspot-jre <span class="se">\</span>
- <span class="o">&&</span> apt-get autoremove -yqq --purge <span class="se">\</span>
- <span class="o">&&</span> apt-get clean <span class="se">\</span>
- <span class="o">&&</span> rm -rf /var/lib/apt/lists/*
-<span class="k">ENV</span> <span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/adoptopenjdk-8-hotspot-jre-amd64
-
-<span class="k">RUN</span> mkdir -p /opt/spark/jars
-
-<span class="c"># Install Apache Hadoop</span>
-<span class="k">ARG</span> <span class="nv">HADOOP_VERSION</span><span class="o">=</span><span class="m">2</span>.10.1
-<span class="k">ENV</span> <span class="nv">HADOOP_HOME</span><span class="o">=</span>/opt/hadoop
-<span class="k">ENV</span> <span class="nv">HADOOP_CONF_DIR</span><span class="o">=</span>/etc/hadoop
-<span class="k">ENV</span> <span class="nv">MULTIHOMED_NETWORK</span><span class="o">=</span><span class="m">1</span>
-<span class="k">ENV</span> <span class="nv">USER</span><span class="o">=</span>root
-
-<span class="k">RUN</span> <span class="nv">HADOOP_URL</span><span class="o">=</span><span class="s2">"https://archive.apache.org/dist/hadoop/common/hadoop-</span><span class="nv">$HADOOP_VERSION</span><span class="s2">/hadoop-</span><span class="nv">$HADOOP_VERSION</span><span class="s2">.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> curl <span class="s1">'https://dist.apache.org/repos/dist/release/hadoop/common/KEYS'</span> <span class="p">|</span> gpg --import - <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HADOOP_URL</span><span class="s2">"</span> -o /tmp/hadoop.tar.gz <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HADOOP_URL</span><span class="s2">.asc"</span> -o /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> gpg --verify /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> tar -xvf /tmp/hadoop.tar.gz -C <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> rm /tmp/hadoop.tar.gz /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> ln -s <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">/etc/hadoop"</span> /etc/hadoop <span class="se">\</span>
- <span class="o">&&</span> mkdir <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">/logs"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir /hadoop-data
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="nv">$HADOOP_HOME</span><span class="s2">/bin/:</span><span class="nv">$PATH</span><span class="s2">"</span>
-
-<span class="c"># Install Apache Hive</span>
-<span class="k">ARG</span> <span class="nv">HIVE_VERSION</span><span class="o">=</span><span class="m">2</span>.3.7
-<span class="k">ENV</span> <span class="nv">HIVE_HOME</span><span class="o">=</span>/opt/hive
-<span class="k">ENV</span> <span class="nv">HIVE_CONF_DIR</span><span class="o">=</span>/etc/hive
-
-<span class="k">RUN</span> <span class="nv">HIVE_URL</span><span class="o">=</span><span class="s2">"https://archive.apache.org/dist/hive/hive-</span><span class="si">${</span><span class="nv">HIVE_VERSION</span><span class="si">}</span><span class="s2">/apache-hive-</span><span class="si">${</span><span class="nv">HIVE_VERSION</span><span class="si">}</span><span class="s2">-bin.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s1">'https://downloads.apache.org/hive/KEYS'</span> <span class="p">|</span> gpg --import - <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HIVE_URL</span><span class="s2">"</span> -o /tmp/hive.tar.gz <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HIVE_URL</span><span class="s2">.asc"</span> -o /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> gpg --verify /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> mkdir -p <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> tar -xf /tmp/hive.tar.gz -C <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> rm /tmp/hive.tar.gz /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> ln -s <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">/etc/hive"</span> <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_CONF_DIR</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">/logs"</span>
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="nv">$HIVE_HOME</span><span class="s2">/bin/:</span><span class="nv">$PATH</span><span class="s2">"</span>
-
-<span class="c"># Install GCS connector for Apache Hadoop</span>
-<span class="c"># See: https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage</span>
-<span class="k">ARG</span> <span class="nv">GCS_VARIANT</span><span class="o">=</span><span class="s2">"hadoop2"</span>
-<span class="k">ARG</span> <span class="nv">GCS_VERSION</span><span class="o">=</span><span class="s2">"2.1.5"</span>
-
-<span class="k">RUN</span> <span class="nv">GCS_JAR_PATH</span><span class="o">=</span><span class="s2">"/opt/spark/jars/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar"</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="nv">GCS_JAR_URL</span><span class="o">=</span><span class="s2">"https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar"</span> <span class="se">\</span>
- <span class="o">&&</span> curl <span class="s2">"</span><span class="si">${</span><span class="nv">GCS_JAR_URL</span><span class="si">}</span><span class="s2">"</span> -o <span class="s2">"</span><span class="si">${</span><span class="nv">GCS_JAR_PATH</span><span class="si">}</span><span class="s2">"</span>
-
-<span class="k">ENV</span> <span class="nv">HADOOP_CLASSPATH</span><span class="o">=</span><span class="s2">"/opt/spark/jars/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar:</span><span class="nv">$HADOOP_CLASSPATH</span><span class="s2">"</span>
-
-<span class="k">USER</span><span class="s"> ${AIRFLOW_UID}</span>
-</pre></div>
-</div>
-</div>
-<p>Then build a new image.</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">BASE_AIRFLOW_IMAGE</span><span class="o">=</span><span class="s2">"apache/airflow:2.0.2"</span> <span class="se">\</span>
- -t my-airflow-image
-</pre></div>
+ <div class="section" id="module-airflow.executors.debug_executor">
+<span id="airflow-executors-debug-executor"></span><h1><a class="reference internal" href="#module-airflow.executors.debug_executor" title="airflow.executors.debug_executor"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a><a class="headerlink" href="#module-airflow.executors.debug_executor" title="Permalink to this headline">¶</a></h1>
+<p><p>DebugExecutor</p>
+<div class="admonition seealso">
+<p class="admonition-title">See also</p>
+<p>For more information on how the DebugExecutor works, take a look at the guide:
+<a class="reference internal" href="../../../../executor/debug.html#executor-debugexecutor"><span class="std std-ref">Debug Executor</span></a></p>
</div>
+</p>
+<div class="section" id="module-contents">
+<h2>Module Contents<a class="headerlink" href="#module-contents" title="Permalink to this headline">¶</a></h2>
+<dl class="py class">
+<dt id="airflow.executors.debug_executor.DebugExecutor">
+<em class="property">class </em><code class="sig-prename descclassname">airflow.executors.debug_executor.</code><code class="sig-name descname">DebugExecutor</code><a class="reference internal" href="../../../../_modules/airflow/executors/debug_executor.html#DebugExecutor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.debug_executor.DebugExecutor" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="../base_executor/index.html#airflow.executors.base_executor.BaseExecutor" title="airflow.executors.base_executor.BaseExecutor"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.executors.base_executor.BaseExecutor</span></code></a></p>
+<p>This executor is meant for debugging purposes. It can be used with SQLite.</p>
+<p>It executes one task instance at time. Additionally to support working
+with sensors, all sensors <code class="docutils literal notranslate"><span class="pre">mode</span></code> will be automatically set to "reschedule".</p>
+<dl class="py attribute">
+<dt id="airflow.executors.debug_executor.DebugExecutor._terminated">
+<code class="sig-name descname">_terminated</code><a class="reference internal" href="../../../../_modules/airflow/executors/debug_executor.html#DebugExecutor._terminated"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.debug_executor.DebugExecutor._terminated" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.debug_executor.DebugExecutor.execute_async">
+<code class="sig-name descname">execute_async</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="o">*</span><span class="n">args</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/debug_executor.html#DebugExecutor.execute_async"><span class="viewcode-link">[source [...]
+<dd><p>The method is replaced by custom trigger_task implementation.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.debug_executor.DebugExecutor.sync">
+<code class="sig-name descname">sync</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/debug_executor.html#DebugExecutor.sync"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.debug_executor.DebugExecutor.sync" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.debug_executor.DebugExecutor._run_task">
+<code class="sig-name descname">_run_task</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">ti</span><span class="p">:</span> <span class="n">TaskInstance</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/debug_executor.html#DebugExecutor._run_task"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executor [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.debug_executor.DebugExecutor.queue_task_instance">
+<code class="sig-name descname">queue_task_instance</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">task_instance</span><span class="p">:</span> <span class="n">TaskInstance</span></em>, <em class="sig-param"><span class="n">mark_success</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.9)">bool</a [...]
+<dd><p>Queues task instance with empty command because we do not need it.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.debug_executor.DebugExecutor.trigger_tasks">
+<code class="sig-name descname">trigger_tasks</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">open_slots</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.9)">int</a></span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/debug_executor. [...]
+<dd><p>Triggers tasks. Instead of calling exec_async we just
+add task instance to tasks_to_run queue.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>open_slots</strong> -- Number of open slots</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.debug_executor.DebugExecutor.end">
+<code class="sig-name descname">end</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/debug_executor.html#DebugExecutor.end"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.debug_executor.DebugExecutor.end" title="Permalink to this definition">¶</a></dt>
+<dd><p>When the method is called we just set states of queued tasks
+to UPSTREAM_FAILED marking them as not executed.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.debug_executor.DebugExecutor.terminate">
+<code class="sig-name descname">terminate</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/debug_executor.html#DebugExecutor.terminate"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.debug_executor.DebugExecutor.terminate" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.debug_executor.DebugExecutor.change_state">
+<code class="sig-name descname">change_state</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">key</span><span class="p">:</span> <span class="n">TaskInstanceKey</span></em>, <em class="sig-param"><span class="n">state</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a></span></em>, <em class [...]
+<dd></dd></dl>
+
+</dd></dl>
+
</div>
</div>
<div class="pager" role="navigation" aria-label="related navigation">
- <a rel="prev" title="Entrypoint" href="entrypoint.html" >
+ <a rel="prev" title="airflow.executors.dask_executor" href="../dask_executor/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="p">Previous</button>
</a>
- <a rel="next" title="Image build arguments reference" href="build-arg-ref.html" >
+ <a rel="next" title="airflow.executors.executor_constants" href="../executor_constants/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="n">Next</button>
</a>
</div>
@@ -819,9 +909,38 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
<nav class="wy-nav-side-toc">
<div class="wy-menu-vertical">
<ul>
-<li><a class="reference internal" href="#">Recipes</a><ul>
-<li><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
+<li><a class="reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a><ul>
+<li><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor">DebugExecutor</a><ul>
+<li><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor._terminated">_terminated</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.execute_async">execute_async</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.sync">sync</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor._run_task">_run_task</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.queue_task_instance">queue_task_instance</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.trigger_tasks">trigger_tasks</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.end">end</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.terminate">terminate</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.debug_executor.DebugExecutor.change_state">change_state</a></li>
+</ul>
+</li>
+</ul>
+</li>
</ul>
</li>
</ul>
@@ -846,7 +965,7 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
<div class="base-layout--button">
- <a href="https://github.com/apache/airflow/edit/devel/docs/docker-stack/recipes.rst" rel="nofollow">
+ <a href="https://github.com/apache/airflow/edit/devel/docs/apache-airflow/_api/airflow/executors/debug_executor/index.rst" rel="nofollow">
<button class="btn-hollow btn-brown btn-with-icon with-box-shadow button-fixed">
<svg xmlns="http://www.w3.org/2000/svg" width="30.76" height="30">
@@ -980,15 +1099,16 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
</span>
</div>
</footer>
- <script type="text/javascript" src="_static/_gen/js/docs.js"></script>
- <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/_gen/js/docs.js"></script>
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
<script src="https://code.jquery.com/jquery-3.4.1.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script>
- <script src="_static/jquery.js"></script>
- <script src="_static/underscore.js"></script>
- <script src="_static/doctools.js"></script>
- <script src="_static/clipboard.min.js"></script>
- <script src="_static/copybutton.js"></script>
+ <script src="../../../../_static/jquery.js"></script>
+ <script src="../../../../_static/underscore.js"></script>
+ <script src="../../../../_static/doctools.js"></script>
+ <script src="../../../../_static/clipboard.min.js"></script>
+ <script src="../../../../_static/copybutton.js"></script>
+ <script src="../../../../_static/jira-links.js"></script>
</body>
</html>
\ No newline at end of file
diff --git a/docs-archive/docker-stack/recipes.html b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/executor_constants/index.html
similarity index 74%
copy from docs-archive/docker-stack/recipes.html
copy to docs-archive/apache-airflow/2.1.0/_api/airflow/executors/executor_constants/index.html
index 75019af..0af7637 100644
--- a/docs-archive/docker-stack/recipes.html
+++ b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/executor_constants/index.html
@@ -9,15 +9,16 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Recipes — docker-stack Documentation</title>
- <link rel="stylesheet" href="_static/_gen/css/main.min.css" type="text/css" />
- <link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
- <link rel="stylesheet" type="text/css" href="_static/_gen/css/main-custom.min.css" />
- <link rel="shortcut icon" href="_static/pin_32.png"/>
- <link rel="index" title="Index" href="genindex.html" />
- <link rel="search" title="Search" href="search.html" />
- <link rel="next" title="Image build arguments reference" href="build-arg-ref.html" />
- <link rel="prev" title="Entrypoint" href="entrypoint.html" />
+ <title>airflow.executors.executor_constants — Airflow Documentation</title>
+ <link rel="stylesheet" href="../../../../_static/_gen/css/main.min.css" type="text/css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/copybutton.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/_gen/css/main-custom.min.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/graphviz.css" />
+ <link rel="shortcut icon" href="../../../../_static/pin_32.png"/>
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+ <link rel="next" title="airflow.executors.executor_loader" href="../executor_loader/index.html" />
+ <link rel="prev" title="airflow.executors.debug_executor" href="../debug_executor/index.html" />
<script type="application/javascript">
var doNotTrack = false;
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
@@ -272,7 +273,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -283,7 +284,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -313,19 +314,103 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Recipes</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
-</ul>
-</li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6"><a class="reference internal" href="../base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_constants.LOCAL_EXECUTOR">LOCAL_EXECUTOR</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_constants.SEQUENTIAL_EXECUTOR">SEQUENTIAL_EXECUTOR</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_constants.CELERY_EXECUTOR">CELERY_EXECUTOR</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_constants.CELERY_KUBERNETES_EXECUTOR">CELERY_KUBERNETES_EXECUTOR</a></li>
+</ul>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_constants.DASK_EXECUTOR">DASK_EXECUTOR</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_constants.KUBERNETES_EXECUTOR">KUBERNETES_EXECUTOR</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_constants.DEBUG_EXECUTOR">DEBUG_EXECUTOR</a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -409,7 +494,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -420,7 +505,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -450,19 +535,103 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Recipes</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
-</ul>
-</li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6"><a class="reference internal" href="../base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_constants.LOCAL_EXECUTOR">LOCAL_EXECUTOR</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_constants.SEQUENTIAL_EXECUTOR">SEQUENTIAL_EXECUTOR</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_constants.CELERY_EXECUTOR">CELERY_EXECUTOR</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_constants.CELERY_KUBERNETES_EXECUTOR">CELERY_KUBERNETES_EXECUTOR</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_constants.DASK_EXECUTOR">DASK_EXECUTOR</a></li>
+</ul>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_constants.KUBERNETES_EXECUTOR">KUBERNETES_EXECUTOR</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_constants.DEBUG_EXECUTOR">DEBUG_EXECUTOR</a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -551,9 +720,13 @@
<ul class="breadcrumb">
- <li class="breadcrumb-item"><a href="index.html" class="icon icon-home"> Home</a></li>
+ <li class="breadcrumb-item"><a href="../../../../index.html" class="icon icon-home"> Home</a></li>
+
+ <li class="breadcrumb-item"><a href="../../../../python-api-ref.html">Python API Reference</a></li>
+
+ <li class="breadcrumb-item"><a href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a></li>
- <li class="breadcrumb-item"><a href="recipes.html"> Recipes</a></li>
+ <li class="breadcrumb-item"><a href="index.html"> <code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
</ul>
</div>
@@ -564,192 +737,54 @@
<div class="bodywrapper">
<div class="body" role="main">
- <blockquote>
-<div></div></blockquote>
-<div class="section" id="recipes">
-<h1>Recipes<a class="headerlink" href="#recipes" title="Permalink to this headline">¶</a></h1>
-<p>Users sometimes share interesting ways of using the Docker images. We encourage users to contribute these
-recipes to the documentation in case they prove useful to other members of the community by
-submitting a pull request. The sections below capture this knowledge.</p>
-<div class="section" id="google-cloud-sdk-installation">
-<h2>Google Cloud SDK installation<a class="headerlink" href="#google-cloud-sdk-installation" title="Permalink to this headline">¶</a></h2>
-<p>Some operators, such as <a class="reference external" href="/docs/apache-airflow-providers-google/stable/_api/airflow/providers/google/cloud/operators/kubernetes_engine/index.html#airflow.providers.google.cloud.operators.kubernetes_engine.GKEStartPodOperator" title="(in apache-airflow-providers-google vdevel)"><code class="xref py py-class docutils literal notranslate"><span class="pre">GKEStartPodOperator</span></code></a>,
-<a class="reference external" href="/docs/apache-airflow-providers-google/stable/_api/airflow/providers/google/cloud/operators/dataflow/index.html#airflow.providers.google.cloud.operators.dataflow.DataflowStartSqlJobOperator" title="(in apache-airflow-providers-google vdevel)"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataflowStartSqlJobOperator</span></code></a>, require
-the installation of <a class="reference external" href="https://cloud.google.com/sdk">Google Cloud SDK</a> (includes <code class="docutils literal notranslate"><span class="pre">gcloud</span></code>).
-You can also run these commands with BashOperator.</p>
-<p>Create a new Dockerfile like the one shown below.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-images-recipes/gcloud.Dockerfile</span></p>
-<div class="highlight-dockerfile notranslate"><div class="highlight"><pre><span></span><span class="c"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
-<span class="c"># contributor license agreements. See the NOTICE file distributed with</span>
-<span class="c"># this work for additional information regarding copyright ownership.</span>
-<span class="c"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
-<span class="c"># (the "License"); you may not use this file except in compliance with</span>
-<span class="c"># the License. You may obtain a copy of the License at</span>
-<span class="c">#</span>
-<span class="c"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c">#</span>
-<span class="c"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c"># See the License for the specific language governing permissions and</span>
-<span class="c"># limitations under the License.</span>
-<span class="k">ARG</span> BASE_AIRFLOW_IMAGE
-<span class="k">FROM</span> <span class="s">${BASE_AIRFLOW_IMAGE}</span>
-
-<span class="k">SHELL</span> <span class="p">[</span><span class="s2">"/bin/bash"</span><span class="p">,</span> <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"pipefail"</span><span class="p">,</span> <span class="s2">"-e"</span><span class="p">,</span> <span class="s2">"-u"</span><span class="p">,</span> <span class="s2">"-x"</span><span class="p">,</span> <span class="s2">"-c"</span><span clas [...]
-
-<span class="k">USER</span><span class="s"> 0</span>
-
-<span class="k">ARG</span> <span class="nv">CLOUD_SDK_VERSION</span><span class="o">=</span><span class="m">322</span>.0.0
-<span class="k">ENV</span> <span class="nv">GCLOUD_HOME</span><span class="o">=</span>/opt/google-cloud-sdk
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">/bin/:</span><span class="si">${</span><span class="nv">PATH</span><span class="si">}</span><span class="s2">"</span>
-
-<span class="k">RUN</span> <span class="nv">DOWNLOAD_URL</span><span class="o">=</span><span class="s2">"https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-</span><span class="si">${</span><span class="nv">CLOUD_SDK_VERSION</span><span class="si">}</span><span class="s2">-linux-x86_64.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="nv">TMP_DIR</span><span class="o">=</span><span class="s2">"</span><span class="k">$(</span>mktemp -d<span class="k">)</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> curl -fL <span class="s2">"</span><span class="si">${</span><span class="nv">DOWNLOAD_URL</span><span class="si">}</span><span class="s2">"</span> --output <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">/google-cloud-sdk.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir -p <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> tar xzf <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">/google-cloud-sdk.tar.gz"</span> -C <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="s2">"</span><span class="si">${</span><span class="nv">GCLOUD_HOME</span><span class="si">}</span><span class="s2">/install.sh"</span> <span class="se">\</span>
- --bash-completion<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --path-update<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --usage-reporting<span class="o">=</span><span class="nb">false</span> <span class="se">\</span>
- --additional-components alpha beta kubectl <span class="se">\</span>
- --quiet <span class="se">\</span>
- <span class="o">&&</span> rm -rf <span class="s2">"</span><span class="si">${</span><span class="nv">TMP_DIR</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> gcloud --version
-
-<span class="k">USER</span><span class="s"> ${AIRFLOW_UID}</span>
-</pre></div>
-</div>
-</div>
-<p>Then build a new image.</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">BASE_AIRFLOW_IMAGE</span><span class="o">=</span><span class="s2">"apache/airflow:2.0.2"</span> <span class="se">\</span>
- -t my-airflow-image
-</pre></div>
-</div>
-</div>
-<div class="section" id="apache-hadoop-stack-installation">
-<h2>Apache Hadoop Stack installation<a class="headerlink" href="#apache-hadoop-stack-installation" title="Permalink to this headline">¶</a></h2>
-<p>Airflow is often used to run tasks on Hadoop cluster. It required Java Runtime Environment (JRE) to run.
-Below are the steps to take tools that are frequently used in Hadoop-world:</p>
-<ul class="simple">
-<li><p>Java Runtime Environment (JRE)</p></li>
-<li><p>Apache Hadoop</p></li>
-<li><p>Apache Hive</p></li>
-<li><p><a class="reference external" href="https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage">Cloud Storage connector for Apache Hadoop</a></p></li>
-</ul>
-<p>Create a new Dockerfile like the one shown below.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-images-recipes/hadoop.Dockerfile</span></p>
-<div class="highlight-dockerfile notranslate"><div class="highlight"><pre><span></span><span class="c"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
-<span class="c"># contributor license agreements. See the NOTICE file distributed with</span>
-<span class="c"># this work for additional information regarding copyright ownership.</span>
-<span class="c"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
-<span class="c"># (the "License"); you may not use this file except in compliance with</span>
-<span class="c"># the License. You may obtain a copy of the License at</span>
-<span class="c">#</span>
-<span class="c"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c">#</span>
-<span class="c"># Unless required by applicable law or agreed to in writing, software</span>
-<span class="c"># distributed under the License is distributed on an "AS IS" BASIS,</span>
-<span class="c"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
-<span class="c"># See the License for the specific language governing permissions and</span>
-<span class="c"># limitations under the License.</span>
-<span class="k">ARG</span> BASE_AIRFLOW_IMAGE
-
-<span class="k">FROM</span> <span class="s">${BASE_AIRFLOW_IMAGE}</span>
-
-<span class="k">SHELL</span> <span class="p">[</span><span class="s2">"/bin/bash"</span><span class="p">,</span> <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"pipefail"</span><span class="p">,</span> <span class="s2">"-e"</span><span class="p">,</span> <span class="s2">"-u"</span><span class="p">,</span> <span class="s2">"-x"</span><span class="p">,</span> <span class="s2">"-c"</span><span clas [...]
-
-<span class="k">USER</span><span class="s"> 0</span>
-
-<span class="c"># Install Java</span>
-<span class="k">RUN</span> mkdir -pv /usr/share/man/man1 <span class="se">\</span>
- <span class="o">&&</span> mkdir -pv /usr/share/man/man7 <span class="se">\</span>
- <span class="o">&&</span> curl -fsSL https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public <span class="p">|</span> apt-key add - <span class="se">\</span>
- <span class="o">&&</span> <span class="nb">echo</span> <span class="s1">'deb https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ buster main'</span> > <span class="se">\</span>
- /etc/apt/sources.list.d/adoptopenjdk.list <span class="se">\</span>
- <span class="o">&&</span> apt-get update <span class="se">\</span>
- <span class="o">&&</span> apt-get install --no-install-recommends -y <span class="se">\</span>
- adoptopenjdk-8-hotspot-jre <span class="se">\</span>
- <span class="o">&&</span> apt-get autoremove -yqq --purge <span class="se">\</span>
- <span class="o">&&</span> apt-get clean <span class="se">\</span>
- <span class="o">&&</span> rm -rf /var/lib/apt/lists/*
-<span class="k">ENV</span> <span class="nv">JAVA_HOME</span><span class="o">=</span>/usr/lib/jvm/adoptopenjdk-8-hotspot-jre-amd64
-
-<span class="k">RUN</span> mkdir -p /opt/spark/jars
-
-<span class="c"># Install Apache Hadoop</span>
-<span class="k">ARG</span> <span class="nv">HADOOP_VERSION</span><span class="o">=</span><span class="m">2</span>.10.1
-<span class="k">ENV</span> <span class="nv">HADOOP_HOME</span><span class="o">=</span>/opt/hadoop
-<span class="k">ENV</span> <span class="nv">HADOOP_CONF_DIR</span><span class="o">=</span>/etc/hadoop
-<span class="k">ENV</span> <span class="nv">MULTIHOMED_NETWORK</span><span class="o">=</span><span class="m">1</span>
-<span class="k">ENV</span> <span class="nv">USER</span><span class="o">=</span>root
-
-<span class="k">RUN</span> <span class="nv">HADOOP_URL</span><span class="o">=</span><span class="s2">"https://archive.apache.org/dist/hadoop/common/hadoop-</span><span class="nv">$HADOOP_VERSION</span><span class="s2">/hadoop-</span><span class="nv">$HADOOP_VERSION</span><span class="s2">.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> curl <span class="s1">'https://dist.apache.org/repos/dist/release/hadoop/common/KEYS'</span> <span class="p">|</span> gpg --import - <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HADOOP_URL</span><span class="s2">"</span> -o /tmp/hadoop.tar.gz <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HADOOP_URL</span><span class="s2">.asc"</span> -o /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> gpg --verify /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> tar -xvf /tmp/hadoop.tar.gz -C <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> rm /tmp/hadoop.tar.gz /tmp/hadoop.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> ln -s <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">/etc/hadoop"</span> /etc/hadoop <span class="se">\</span>
- <span class="o">&&</span> mkdir <span class="s2">"</span><span class="si">${</span><span class="nv">HADOOP_HOME</span><span class="si">}</span><span class="s2">/logs"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir /hadoop-data
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="nv">$HADOOP_HOME</span><span class="s2">/bin/:</span><span class="nv">$PATH</span><span class="s2">"</span>
-
-<span class="c"># Install Apache Hive</span>
-<span class="k">ARG</span> <span class="nv">HIVE_VERSION</span><span class="o">=</span><span class="m">2</span>.3.7
-<span class="k">ENV</span> <span class="nv">HIVE_HOME</span><span class="o">=</span>/opt/hive
-<span class="k">ENV</span> <span class="nv">HIVE_CONF_DIR</span><span class="o">=</span>/etc/hive
-
-<span class="k">RUN</span> <span class="nv">HIVE_URL</span><span class="o">=</span><span class="s2">"https://archive.apache.org/dist/hive/hive-</span><span class="si">${</span><span class="nv">HIVE_VERSION</span><span class="si">}</span><span class="s2">/apache-hive-</span><span class="si">${</span><span class="nv">HIVE_VERSION</span><span class="si">}</span><span class="s2">-bin.tar.gz"</span> <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s1">'https://downloads.apache.org/hive/KEYS'</span> <span class="p">|</span> gpg --import - <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HIVE_URL</span><span class="s2">"</span> -o /tmp/hive.tar.gz <span class="se">\</span>
- <span class="o">&&</span> curl -fSL <span class="s2">"</span><span class="nv">$HIVE_URL</span><span class="s2">.asc"</span> -o /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> gpg --verify /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> mkdir -p <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> tar -xf /tmp/hive.tar.gz -C <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">"</span> --strip-components<span class="o">=</span><span class="m">1</span> <span class="se">\</span>
- <span class="o">&&</span> rm /tmp/hive.tar.gz /tmp/hive.tar.gz.asc <span class="se">\</span>
- <span class="o">&&</span> ln -s <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">/etc/hive"</span> <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_CONF_DIR</span><span class="si">}</span><span class="s2">"</span> <span class="se">\</span>
- <span class="o">&&</span> mkdir <span class="s2">"</span><span class="si">${</span><span class="nv">HIVE_HOME</span><span class="si">}</span><span class="s2">/logs"</span>
-
-<span class="k">ENV</span> <span class="nv">PATH</span><span class="o">=</span><span class="s2">"</span><span class="nv">$HIVE_HOME</span><span class="s2">/bin/:</span><span class="nv">$PATH</span><span class="s2">"</span>
-
-<span class="c"># Install GCS connector for Apache Hadoop</span>
-<span class="c"># See: https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage</span>
-<span class="k">ARG</span> <span class="nv">GCS_VARIANT</span><span class="o">=</span><span class="s2">"hadoop2"</span>
-<span class="k">ARG</span> <span class="nv">GCS_VERSION</span><span class="o">=</span><span class="s2">"2.1.5"</span>
-
-<span class="k">RUN</span> <span class="nv">GCS_JAR_PATH</span><span class="o">=</span><span class="s2">"/opt/spark/jars/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar"</span> <span class="se">\</span>
- <span class="o">&&</span> <span class="nv">GCS_JAR_URL</span><span class="o">=</span><span class="s2">"https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar"</span> <span class="se">\</span>
- <span class="o">&&</span> curl <span class="s2">"</span><span class="si">${</span><span class="nv">GCS_JAR_URL</span><span class="si">}</span><span class="s2">"</span> -o <span class="s2">"</span><span class="si">${</span><span class="nv">GCS_JAR_PATH</span><span class="si">}</span><span class="s2">"</span>
-
-<span class="k">ENV</span> <span class="nv">HADOOP_CLASSPATH</span><span class="o">=</span><span class="s2">"/opt/spark/jars/gcs-connector-</span><span class="si">${</span><span class="nv">GCS_VARIANT</span><span class="si">}</span><span class="s2">-</span><span class="si">${</span><span class="nv">GCS_VERSION</span><span class="si">}</span><span class="s2">.jar:</span><span class="nv">$HADOOP_CLASSPATH</span><span class="s2">"</span>
-
-<span class="k">USER</span><span class="s"> ${AIRFLOW_UID}</span>
-</pre></div>
-</div>
-</div>
-<p>Then build a new image.</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">BASE_AIRFLOW_IMAGE</span><span class="o">=</span><span class="s2">"apache/airflow:2.0.2"</span> <span class="se">\</span>
- -t my-airflow-image
-</pre></div>
-</div>
+ <div class="section" id="module-airflow.executors.executor_constants">
+<span id="airflow-executors-executor-constants"></span><h1><a class="reference internal" href="#module-airflow.executors.executor_constants" title="airflow.executors.executor_constants"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a><a class="headerlink" href="#module-airflow.executors.executor_constants" title="Permalink to this headline">¶</a></h1>
+<div class="section" id="module-contents">
+<h2>Module Contents<a class="headerlink" href="#module-contents" title="Permalink to this headline">¶</a></h2>
+<dl class="py data">
+<dt id="airflow.executors.executor_constants.LOCAL_EXECUTOR">
+<code class="sig-prename descclassname">airflow.executors.executor_constants.</code><code class="sig-name descname">LOCAL_EXECUTOR</code><em class="property"> = LocalExecutor</em><a class="reference internal" href="../../../../_modules/airflow/executors/executor_constants.html#LOCAL_EXECUTOR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.executor_constants.LOCAL_EXECUTOR" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.executor_constants.SEQUENTIAL_EXECUTOR">
+<code class="sig-prename descclassname">airflow.executors.executor_constants.</code><code class="sig-name descname">SEQUENTIAL_EXECUTOR</code><em class="property"> = SequentialExecutor</em><a class="reference internal" href="../../../../_modules/airflow/executors/executor_constants.html#SEQUENTIAL_EXECUTOR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.executor_constants.SEQUENTIAL_EXECUTOR" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.executor_constants.CELERY_EXECUTOR">
+<code class="sig-prename descclassname">airflow.executors.executor_constants.</code><code class="sig-name descname">CELERY_EXECUTOR</code><em class="property"> = CeleryExecutor</em><a class="reference internal" href="../../../../_modules/airflow/executors/executor_constants.html#CELERY_EXECUTOR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.executor_constants.CELERY_EXECUTOR" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.executor_constants.CELERY_KUBERNETES_EXECUTOR">
+<code class="sig-prename descclassname">airflow.executors.executor_constants.</code><code class="sig-name descname">CELERY_KUBERNETES_EXECUTOR</code><em class="property"> = CeleryKubernetesExecutor</em><a class="reference internal" href="../../../../_modules/airflow/executors/executor_constants.html#CELERY_KUBERNETES_EXECUTOR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.executor_constants.CELERY_KUBERNETES_EXECUTOR" title="Permalink to th [...]
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.executor_constants.DASK_EXECUTOR">
+<code class="sig-prename descclassname">airflow.executors.executor_constants.</code><code class="sig-name descname">DASK_EXECUTOR</code><em class="property"> = DaskExecutor</em><a class="reference internal" href="../../../../_modules/airflow/executors/executor_constants.html#DASK_EXECUTOR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.executor_constants.DASK_EXECUTOR" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.executor_constants.KUBERNETES_EXECUTOR">
+<code class="sig-prename descclassname">airflow.executors.executor_constants.</code><code class="sig-name descname">KUBERNETES_EXECUTOR</code><em class="property"> = KubernetesExecutor</em><a class="reference internal" href="../../../../_modules/airflow/executors/executor_constants.html#KUBERNETES_EXECUTOR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.executor_constants.KUBERNETES_EXECUTOR" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.executor_constants.DEBUG_EXECUTOR">
+<code class="sig-prename descclassname">airflow.executors.executor_constants.</code><code class="sig-name descname">DEBUG_EXECUTOR</code><em class="property"> = DebugExecutor</em><a class="reference internal" href="../../../../_modules/airflow/executors/executor_constants.html#DEBUG_EXECUTOR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.executor_constants.DEBUG_EXECUTOR" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
</div>
</div>
<div class="pager" role="navigation" aria-label="related navigation">
- <a rel="prev" title="Entrypoint" href="entrypoint.html" >
+ <a rel="prev" title="airflow.executors.debug_executor" href="../debug_executor/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="p">Previous</button>
</a>
- <a rel="next" title="Image build arguments reference" href="build-arg-ref.html" >
+ <a rel="next" title="airflow.executors.executor_loader" href="../executor_loader/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="n">Next</button>
</a>
</div>
@@ -819,9 +854,29 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
<nav class="wy-nav-side-toc">
<div class="wy-menu-vertical">
<ul>
-<li><a class="reference internal" href="#">Recipes</a><ul>
-<li><a class="reference internal" href="#google-cloud-sdk-installation">Google Cloud SDK installation</a></li>
-<li><a class="reference internal" href="#apache-hadoop-stack-installation">Apache Hadoop Stack installation</a></li>
+<li><a class="reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a><ul>
+<li><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li><a class="reference internal" href="#airflow.executors.executor_constants.LOCAL_EXECUTOR">LOCAL_EXECUTOR</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.executor_constants.SEQUENTIAL_EXECUTOR">SEQUENTIAL_EXECUTOR</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.executor_constants.CELERY_EXECUTOR">CELERY_EXECUTOR</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.executor_constants.CELERY_KUBERNETES_EXECUTOR">CELERY_KUBERNETES_EXECUTOR</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.executor_constants.DASK_EXECUTOR">DASK_EXECUTOR</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.executor_constants.KUBERNETES_EXECUTOR">KUBERNETES_EXECUTOR</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.executor_constants.DEBUG_EXECUTOR">DEBUG_EXECUTOR</a></li>
+</ul>
+</li>
</ul>
</li>
</ul>
@@ -846,7 +901,7 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
<div class="base-layout--button">
- <a href="https://github.com/apache/airflow/edit/devel/docs/docker-stack/recipes.rst" rel="nofollow">
+ <a href="https://github.com/apache/airflow/edit/devel/docs/apache-airflow/_api/airflow/executors/executor_constants/index.rst" rel="nofollow">
<button class="btn-hollow btn-brown btn-with-icon with-box-shadow button-fixed">
<svg xmlns="http://www.w3.org/2000/svg" width="30.76" height="30">
@@ -980,15 +1035,16 @@ Below are the steps to take tools that are frequently used in Hadoop-world:</p>
</span>
</div>
</footer>
- <script type="text/javascript" src="_static/_gen/js/docs.js"></script>
- <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/_gen/js/docs.js"></script>
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
<script src="https://code.jquery.com/jquery-3.4.1.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script>
- <script src="_static/jquery.js"></script>
- <script src="_static/underscore.js"></script>
- <script src="_static/doctools.js"></script>
- <script src="_static/clipboard.min.js"></script>
- <script src="_static/copybutton.js"></script>
+ <script src="../../../../_static/jquery.js"></script>
+ <script src="../../../../_static/underscore.js"></script>
+ <script src="../../../../_static/doctools.js"></script>
+ <script src="../../../../_static/clipboard.min.js"></script>
+ <script src="../../../../_static/copybutton.js"></script>
+ <script src="../../../../_static/jira-links.js"></script>
</body>
</html>
\ No newline at end of file
diff --git a/docs-archive/docker-stack/entrypoint.html b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/executor_loader/index.html
similarity index 73%
copy from docs-archive/docker-stack/entrypoint.html
copy to docs-archive/apache-airflow/2.1.0/_api/airflow/executors/executor_loader/index.html
index aee6e73..9d17744 100644
--- a/docs-archive/docker-stack/entrypoint.html
+++ b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/executor_loader/index.html
@@ -9,15 +9,16 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Entrypoint — docker-stack Documentation</title>
- <link rel="stylesheet" href="_static/_gen/css/main.min.css" type="text/css" />
- <link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
- <link rel="stylesheet" type="text/css" href="_static/_gen/css/main-custom.min.css" />
- <link rel="shortcut icon" href="_static/pin_32.png"/>
- <link rel="index" title="Index" href="genindex.html" />
- <link rel="search" title="Search" href="search.html" />
- <link rel="next" title="Recipes" href="recipes.html" />
- <link rel="prev" title="Building the image" href="build.html" />
+ <title>airflow.executors.executor_loader — Airflow Documentation</title>
+ <link rel="stylesheet" href="../../../../_static/_gen/css/main.min.css" type="text/css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/copybutton.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/_gen/css/main-custom.min.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/graphviz.css" />
+ <link rel="shortcut icon" href="../../../../_static/pin_32.png"/>
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+ <link rel="next" title="airflow.executors.kubernetes_executor" href="../kubernetes_executor/index.html" />
+ <link rel="prev" title="airflow.executors.executor_constants" href="../executor_constants/index.html" />
<script type="application/javascript">
var doNotTrack = false;
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
@@ -272,7 +273,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -283,7 +284,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -313,22 +314,106 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Entrypoint</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#allowing-arbitrary-user-to-run-the-container">Allowing arbitrary user to run the container</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#waits-for-airflow-db-connection">Waits for Airflow DB connection</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#upgrading-airflow-db">Upgrading Airflow DB</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#creating-admin-user">Creating admin user</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#waits-for-celery-broker-connection">Waits for celery broker connection</a></li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
+</ul>
+<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6"><a class="reference internal" href="../base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_loader.log">log</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader">ExecutorLoader</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader._default_executor">_default_executor</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader.executors">executors</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader.get_default_executor">get_default_executor</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader.load_executor">load_executor</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader.__load_celery_kubernetes_executor">__load_celery_kubernetes_executor</a></li>
</ul>
</li>
-<li class="toctree-l1"><a class="reference internal" href="recipes.html">Recipes</a></li>
</ul>
-<p class="caption"><span class="caption-text">References</span></p>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_loader.UNPICKLEABLE_EXECUTORS">UNPICKLEABLE_EXECUTORS</a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l6"><a class="reference internal" href="../kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -412,7 +497,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -423,7 +508,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -453,22 +538,106 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Entrypoint</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#allowing-arbitrary-user-to-run-the-container">Allowing arbitrary user to run the container</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#waits-for-airflow-db-connection">Waits for Airflow DB connection</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#upgrading-airflow-db">Upgrading Airflow DB</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#creating-admin-user">Creating admin user</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#waits-for-celery-broker-connection">Waits for celery broker connection</a></li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
+</ul>
+<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6"><a class="reference internal" href="../base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_loader.log">log</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader">ExecutorLoader</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader._default_executor">_default_executor</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader.executors">executors</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader.get_default_executor">get_default_executor</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader.load_executor">load_executor</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader.__load_celery_kubernetes_executor">__load_celery_kubernetes_executor</a></li>
</ul>
</li>
-<li class="toctree-l1"><a class="reference internal" href="recipes.html">Recipes</a></li>
</ul>
-<p class="caption"><span class="caption-text">References</span></p>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.executor_loader.UNPICKLEABLE_EXECUTORS">UNPICKLEABLE_EXECUTORS</a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l6"><a class="reference internal" href="../kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -557,9 +726,13 @@
<ul class="breadcrumb">
- <li class="breadcrumb-item"><a href="index.html" class="icon icon-home"> Home</a></li>
+ <li class="breadcrumb-item"><a href="../../../../index.html" class="icon icon-home"> Home</a></li>
- <li class="breadcrumb-item"><a href="entrypoint.html"> Entrypoint</a></li>
+ <li class="breadcrumb-item"><a href="../../../../python-api-ref.html">Python API Reference</a></li>
+
+ <li class="breadcrumb-item"><a href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a></li>
+
+ <li class="breadcrumb-item"><a href="index.html"> <code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
</ul>
</div>
@@ -570,224 +743,78 @@
<div class="bodywrapper">
<div class="body" role="main">
- <blockquote>
-<div></div></blockquote>
-<div class="section" id="entrypoint">
-<h1>Entrypoint<a class="headerlink" href="#entrypoint" title="Permalink to this headline">¶</a></h1>
-<p>If you are using the default entrypoint of the production image,
-there are a few actions that are automatically performed when the container starts.
-In some cases, you can pass environment variables to the image to trigger some of that behaviour.</p>
-<p>The variables that control the "execution" behaviour start with <code class="docutils literal notranslate"><span class="pre">_AIRFLOW</span></code> to distinguish them
-from the variables used to build the image starting with <code class="docutils literal notranslate"><span class="pre">AIRFLOW</span></code>.</p>
-<p>The image entrypoint works as follows:</p>
-<ul>
-<li><p>In case the user is not "airflow" (with undefined user id) and the group id of the user is set to <code class="docutils literal notranslate"><span class="pre">0</span></code> (root),
-then the user is dynamically added to <code class="docutils literal notranslate"><span class="pre">/etc/passwd</span></code> at entry using <code class="docutils literal notranslate"><span class="pre">USER_NAME</span></code> variable to define the user name.
-This is in order to accommodate the
-<a class="reference external" href="https://docs.openshift.com/enterprise/3.0/creating_images/guidelines.html">OpenShift Guidelines</a></p></li>
-<li><p>The <code class="docutils literal notranslate"><span class="pre">AIRFLOW_HOME</span></code> is set by default to <code class="docutils literal notranslate"><span class="pre">/opt/airflow/</span></code> - this means that DAGs
-are in default in the <code class="docutils literal notranslate"><span class="pre">/opt/airflow/dags</span></code> folder and logs are in the <code class="docutils literal notranslate"><span class="pre">/opt/airflow/logs</span></code></p></li>
-<li><p>The working directory is <code class="docutils literal notranslate"><span class="pre">/opt/airflow</span></code> by default.</p></li>
-<li><p>If <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_CONN</span></code> variable is passed to the container and it is either mysql or postgres
-SQL alchemy connection, then the connection is checked and the script waits until the database is reachable.
-If <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_CONN_CMD</span></code> variable is passed to the container, it is evaluated as a
-command to execute and result of this evaluation is used as <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_CONN</span></code>. The
-<code class="docutils literal notranslate"><span class="pre">_CMD</span></code> variable takes precedence over the <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_CONN</span></code> variable.</p></li>
-<li><p>If no <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CORE__SQL_ALCHEMY_CONN</span></code> variable is set then SQLite database is created in
-<code class="docutils literal notranslate"><span class="pre">${AIRFLOW_HOME}/airflow.db</span></code> and db reset is executed.</p></li>
-<li><p>If first argument equals to "bash" - you are dropped to a bash shell or you can executes bash command
-if you specify extra arguments. For example:</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker run -it apache/airflow:master-python3.6 bash -c <span class="s2">"ls -la"</span>
-total <span class="m">16</span>
-drwxr-xr-x <span class="m">4</span> airflow root <span class="m">4096</span> Jun <span class="m">5</span> <span class="m">18</span>:12 .
-drwxr-xr-x <span class="m">1</span> root root <span class="m">4096</span> Jun <span class="m">5</span> <span class="m">18</span>:12 ..
-drwxr-xr-x <span class="m">2</span> airflow root <span class="m">4096</span> Jun <span class="m">5</span> <span class="m">18</span>:12 dags
-drwxr-xr-x <span class="m">2</span> airflow root <span class="m">4096</span> Jun <span class="m">5</span> <span class="m">18</span>:12 logs
-</pre></div>
-</div>
-</li>
-<li><p>If first argument is equal to <code class="docutils literal notranslate"><span class="pre">python</span></code> - you are dropped in python shell or python commands are executed if
-you pass extra parameters. For example:</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>> docker run -it apache/airflow:master-python3.6 python -c <span class="s2">"print('test')"</span>
-<span class="nb">test</span>
-</pre></div>
-</div>
-</li>
-<li><p>If first argument equals to "airflow" - the rest of the arguments is treated as an airflow command
-to execute. Example:</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker run -it apache/airflow:master-python3.6 airflow webserver
-</pre></div>
-</div>
-</li>
-<li><p>If there are any other arguments - they are simply passed to the "airflow" command</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>> docker run -it apache/airflow:master-python3.6 version
-<span class="m">2</span>.1.0.dev0
-</pre></div>
-</div>
-</li>
-<li><p>If <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__BROKER_URL</span></code> variable is passed and airflow command with
-scheduler, worker of flower command is used, then the script checks the broker connection
-and waits until the Celery broker database is reachable.
-If <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__BROKER_URL_CMD</span></code> variable is passed to the container, it is evaluated as a
-command to execute and result of this evaluation is used as <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__BROKER_URL</span></code>. The
-<code class="docutils literal notranslate"><span class="pre">_CMD</span></code> variable takes precedence over the <code class="docutils literal notranslate"><span class="pre">AIRFLOW__CELERY__BROKER_URL</span></code> variable.</p></li>
-</ul>
-<div class="section" id="allowing-arbitrary-user-to-run-the-container">
-<span id="arbitrary-docker-user"></span><h2>Allowing arbitrary user to run the container<a class="headerlink" href="#allowing-arbitrary-user-to-run-the-container" title="Permalink to this headline">¶</a></h2>
-<p>Airflow image is Open-Shift compatible, which means that you can start it with random user ID and the
-group id <code class="docutils literal notranslate"><span class="pre">0</span></code> (<code class="docutils literal notranslate"><span class="pre">root</span></code>). If you want to run the image with user different than Airflow, you MUST set
-GID of the user to <code class="docutils literal notranslate"><span class="pre">0</span></code>. In case you try to use different group, the entrypoint exits with error.</p>
-<p>In order to accommodate a number of external libraries and projects, Airflow will automatically create
-such an arbitrary user in (<cite>/etc/passwd</cite>) and make it's home directory point to <code class="docutils literal notranslate"><span class="pre">/home/airflow</span></code>.
-Many of 3rd-party libraries and packages require home directory of the user to be present, because they
-need to write some cache information there, so such a dynamic creation of a user is necessary.</p>
-<p>Such arbitrary user has to be able to write to certain directories that needs write access, and since
-it is not advised to allow write access to "other" for security reasons, the OpenShift
-guidelines introduced the concept of making all such folders have the <code class="docutils literal notranslate"><span class="pre">0</span></code> (<code class="docutils literal notranslate"><span class="pre">root</span></code>) group id (GID).
-All the directories that need write access in the Airflow production image have GID set to 0 (and
-they are writable for the group). We are following that concept and all the directories that need
-write access follow that.</p>
-<p>The GID=0 is set as default for the <code class="docutils literal notranslate"><span class="pre">airflow</span></code> user, so any directories it creates have GID set to 0
-by default. The entrypoint sets <code class="docutils literal notranslate"><span class="pre">umask</span></code> to be <code class="docutils literal notranslate"><span class="pre">0002</span></code> - this means that any directories created by
-the user have also "group write" access for group <code class="docutils literal notranslate"><span class="pre">0</span></code> - they will be writable by other users with
-<code class="docutils literal notranslate"><span class="pre">root</span></code> group. Also whenever any "arbitrary" user creates a folder (for example in a mounted volume), that
-folder will have a "group write" access and <code class="docutils literal notranslate"><span class="pre">GID=0</span></code>, so that execution with another, arbitrary user
-will still continue to work, even if such directory is mounted by another arbitrary user later.</p>
-<p>The <code class="docutils literal notranslate"><span class="pre">umask</span></code> setting however only works for runtime of the container - it is not used during building of
-the image. If you would like to extend the image and add your own packages, you should remember to add
-<code class="docutils literal notranslate"><span class="pre">umask</span> <span class="pre">0002</span></code> in front of your docker command - this way the directories created by any installation
-that need group access will also be writable for the group. This can be done for example this way:</p>
-<blockquote>
-<div><div class="highlight-docker notranslate"><div class="highlight"><pre><span></span><span class="k">RUN</span> <span class="nb">umask</span> <span class="m">0002</span><span class="p">;</span> <span class="se">\</span>
- do_something<span class="p">;</span> <span class="se">\</span>
- do_otherthing<span class="p">;</span>
-</pre></div>
-</div>
-</div></blockquote>
-<p>You can read more about it in the "Support arbitrary user ids" chapter in the
-<a class="reference external" href="https://docs.openshift.com/container-platform/4.7/openshift_images/create-images.html#images-create-guide-openshift_create-images">Openshift best practices</a>.</p>
-</div>
-<div class="section" id="waits-for-airflow-db-connection">
-<h2>Waits for Airflow DB connection<a class="headerlink" href="#waits-for-airflow-db-connection" title="Permalink to this headline">¶</a></h2>
-<p>In case Postgres or MySQL DB is used, the entrypoint will wait until the airflow DB connection becomes
-available. This happens always when you use the default entrypoint.</p>
-<p>The script detects backend type depending on the URL schema and assigns default port numbers if not specified
-in the URL. Then it loops until the connection to the host/port specified can be established
-It tries <code class="docutils literal notranslate"><span class="pre">CONNECTION_CHECK_MAX_COUNT</span></code> times and sleeps <code class="docutils literal notranslate"><span class="pre">CONNECTION_CHECK_SLEEP_TIME</span></code> between checks
-To disable check, set <code class="docutils literal notranslate"><span class="pre">CONNECTION_CHECK_MAX_COUNT=0</span></code>.</p>
-<p>Supported schemes:</p>
-<ul class="simple">
-<li><p><code class="docutils literal notranslate"><span class="pre">postgres://</span></code> - default port 5432</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">mysql://</span></code> - default port 3306</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">sqlite://</span></code></p></li>
-</ul>
-<p>In case of SQLite backend, there is no connection to establish and waiting is skipped.</p>
-</div>
-<div class="section" id="upgrading-airflow-db">
-<h2>Upgrading Airflow DB<a class="headerlink" href="#upgrading-airflow-db" title="Permalink to this headline">¶</a></h2>
-<p>If you set <code class="docutils literal notranslate"><span class="pre">_AIRFLOW_DB_UPGRADE</span></code> variable to a non-empty value, the entrypoint will run
-the <code class="docutils literal notranslate"><span class="pre">airflow</span> <span class="pre">db</span> <span class="pre">upgrade</span></code> command right after verifying the connection. You can also use this
-when you are running airflow with internal SQLite database (default) to upgrade the db and create
-admin users at entrypoint, so that you can start the webserver immediately. Note - using SQLite is
-intended only for testing purpose, never use SQLite in production as it has severe limitations when it
-comes to concurrency.</p>
-</div>
-<div class="section" id="creating-admin-user">
-<h2>Creating admin user<a class="headerlink" href="#creating-admin-user" title="Permalink to this headline">¶</a></h2>
-<p>The entrypoint can also create webserver user automatically when you enter it. you need to set
-<code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_CREATE</span></code> to a non-empty value in order to do that. This is not intended for
-production, it is only useful if you would like to run a quick test with the production image.
-You need to pass at least password to create such user via <code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_PASSWORD</span></code> or
-<code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_PASSWORD_CMD</span></code> similarly like for other <code class="docutils literal notranslate"><span class="pre">*_CMD</span></code> variables, the content of
-the <code class="docutils literal notranslate"><span class="pre">*_CMD</span></code> will be evaluated as shell command and it's output will be set as password.</p>
-<p>User creation will fail if none of the <code class="docutils literal notranslate"><span class="pre">PASSWORD</span></code> variables are set - there is no default for
-password for security reasons.</p>
-<table class="docutils align-default">
-<colgroup>
-<col style="width: 10%" />
-<col style="width: 24%" />
-<col style="width: 65%" />
-</colgroup>
-<thead>
-<tr class="row-odd"><th class="head"><p>Parameter</p></th>
-<th class="head"><p>Default</p></th>
-<th class="head"><p>Environment variable</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p>username</p></td>
-<td><p>admin</p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_USERNAME</span></code></p></td>
-</tr>
-<tr class="row-odd"><td><p>password</p></td>
-<td></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_PASSWORD_CMD</span></code> or <code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_PASSWORD</span></code></p></td>
-</tr>
-<tr class="row-even"><td><p>firstname</p></td>
-<td><p>Airflow</p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_FIRSTNAME</span></code></p></td>
-</tr>
-<tr class="row-odd"><td><p>lastname</p></td>
-<td><p>Admin</p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_LASTNAME</span></code></p></td>
-</tr>
-<tr class="row-even"><td><p>email</p></td>
-<td><p><a class="reference external" href="mailto:airflowadmin%40example.com">airflowadmin<span>@</span>example<span>.</span>com</a></p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_EMAIL</span></code></p></td>
-</tr>
-<tr class="row-odd"><td><p>role</p></td>
-<td><p>Admin</p></td>
-<td><p><code class="docutils literal notranslate"><span class="pre">_AIRFLOW_WWW_USER_ROLE</span></code></p></td>
-</tr>
-</tbody>
-</table>
-<p>In case the password is specified, the user will be attempted to be created, but the entrypoint will
-not fail if the attempt fails (this accounts for the case that the user is already created).</p>
-<p>You can, for example start the webserver in the production image with initializing the internal SQLite
-database and creating an <code class="docutils literal notranslate"><span class="pre">admin/admin</span></code> Admin user with the following command:</p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker run -it -p <span class="m">8080</span>:8080 <span class="se">\</span>
- --env <span class="s2">"_AIRFLOW_DB_UPGRADE=true"</span> <span class="se">\</span>
- --env <span class="s2">"_AIRFLOW_WWW_USER_CREATE=true"</span> <span class="se">\</span>
- --env <span class="s2">"_AIRFLOW_WWW_USER_PASSWORD=admin"</span> <span class="se">\</span>
- apache/airflow:master-python3.8 webserver
-</pre></div>
-</div>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker run -it -p <span class="m">8080</span>:8080 <span class="se">\</span>
- --env <span class="s2">"_AIRFLOW_DB_UPGRADE=true"</span> <span class="se">\</span>
- --env <span class="s2">"_AIRFLOW_WWW_USER_CREATE=true"</span> <span class="se">\</span>
- --env <span class="s2">"_AIRFLOW_WWW_USER_PASSWORD_CMD=echo admin"</span> <span class="se">\</span>
- apache/airflow:master-python3.8 webserver
-</pre></div>
-</div>
-<p>The commands above perform initialization of the SQLite database, create admin user with admin password
-and Admin role. They also forward local port <code class="docutils literal notranslate"><span class="pre">8080</span></code> to the webserver port and finally start the webserver.</p>
-</div>
-<div class="section" id="waits-for-celery-broker-connection">
-<h2>Waits for celery broker connection<a class="headerlink" href="#waits-for-celery-broker-connection" title="Permalink to this headline">¶</a></h2>
-<p>In case Postgres or MySQL DB is used, and one of the <code class="docutils literal notranslate"><span class="pre">scheduler</span></code>, <code class="docutils literal notranslate"><span class="pre">celery</span></code>, <code class="docutils literal notranslate"><span class="pre">worker</span></code>, or <code class="docutils literal notranslate"><span class="pre">flower</span></code>
-commands are used the entrypoint will wait until the celery broker DB connection is available.</p>
-<p>The script detects backend type depending on the URL schema and assigns default port numbers if not specified
-in the URL. Then it loops until connection to the host/port specified can be established
-It tries <code class="docutils literal notranslate"><span class="pre">CONNECTION_CHECK_MAX_COUNT</span></code> times and sleeps <code class="docutils literal notranslate"><span class="pre">CONNECTION_CHECK_SLEEP_TIME</span></code> between checks.
-To disable check, set <code class="docutils literal notranslate"><span class="pre">CONNECTION_CHECK_MAX_COUNT=0</span></code>.</p>
-<p>Supported schemes:</p>
-<ul class="simple">
-<li><p><code class="docutils literal notranslate"><span class="pre">amqp(s)://</span></code> (rabbitmq) - default port 5672</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">redis://</span></code> - default port 6379</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">postgres://</span></code> - default port 5432</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">mysql://</span></code> - default port 3306</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">sqlite://</span></code></p></li>
-</ul>
-<p>In case of SQLite backend, there is no connection to establish and waiting is skipped.</p>
+ <div class="section" id="module-airflow.executors.executor_loader">
+<span id="airflow-executors-executor-loader"></span><h1><a class="reference internal" href="#module-airflow.executors.executor_loader" title="airflow.executors.executor_loader"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a><a class="headerlink" href="#module-airflow.executors.executor_loader" title="Permalink to this headline">¶</a></h1>
+<p><p>All executors.</p>
+</p>
+<div class="section" id="module-contents">
+<h2>Module Contents<a class="headerlink" href="#module-contents" title="Permalink to this headline">¶</a></h2>
+<dl class="py data">
+<dt id="airflow.executors.executor_loader.log">
+<code class="sig-prename descclassname">airflow.executors.executor_loader.</code><code class="sig-name descname">log</code><a class="reference internal" href="../../../../_modules/airflow/executors/executor_loader.html#log"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.executor_loader.log" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py class">
+<dt id="airflow.executors.executor_loader.ExecutorLoader">
+<em class="property">class </em><code class="sig-prename descclassname">airflow.executors.executor_loader.</code><code class="sig-name descname">ExecutorLoader</code><a class="reference internal" href="../../../../_modules/airflow/executors/executor_loader.html#ExecutorLoader"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.executor_loader.ExecutorLoader" title="Permalink to this definition">¶</a></dt>
+<dd><p>Keeps constants for all the currently available executors.</p>
+<dl class="py attribute">
+<dt id="airflow.executors.executor_loader.ExecutorLoader._default_executor">
+<code class="sig-name descname">_default_executor</code><em class="property"> :Optional[BaseExecutor]</em><a class="reference internal" href="../../../../_modules/airflow/executors/executor_loader.html#ExecutorLoader._default_executor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.executor_loader.ExecutorLoader._default_executor" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py attribute">
+<dt id="airflow.executors.executor_loader.ExecutorLoader.executors">
+<code class="sig-name descname">executors</code><a class="reference internal" href="../../../../_modules/airflow/executors/executor_loader.html#ExecutorLoader.executors"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.executor_loader.ExecutorLoader.executors" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.executor_loader.ExecutorLoader.get_default_executor">
+<em class="property">classmethod </em><code class="sig-name descname">get_default_executor</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">cls</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/executor_loader.html#ExecutorLoader.get_default_executor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.executor_loader.ExecutorLoader.get_default_executor [...]
+<dd><p>Creates a new instance of the configured executor if none exists and returns it</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.executor_loader.ExecutorLoader.load_executor">
+<em class="property">classmethod </em><code class="sig-name descname">load_executor</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">cls</span></em>, <em class="sig-param"><span class="n">executor_name</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a></span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_mo [...]
+<dd><p>Loads the executor.</p>
+<p>This supports the following formats:
+* by executor name for core executor
+* by <code class="docutils literal notranslate"><span class="pre">{plugin_name}.{class_name}</span></code> for executor from plugins
+* by import path.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>an instance of executor class via executor_name</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.executor_loader.ExecutorLoader.__load_celery_kubernetes_executor">
+<em class="property">classmethod </em><code class="sig-name descname">__load_celery_kubernetes_executor</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">cls</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/executor_loader.html#ExecutorLoader.__load_celery_kubernetes_executor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.executor_loader.ExecutorL [...]
+<dd><dl class="field-list simple">
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>an instance of CeleryKubernetesExecutor</p>
+</dd>
+</dl>
+</dd></dl>
+
+</dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.executor_loader.UNPICKLEABLE_EXECUTORS">
+<code class="sig-prename descclassname">airflow.executors.executor_loader.</code><code class="sig-name descname">UNPICKLEABLE_EXECUTORS</code><a class="reference internal" href="../../../../_modules/airflow/executors/executor_loader.html#UNPICKLEABLE_EXECUTORS"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.executor_loader.UNPICKLEABLE_EXECUTORS" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
</div>
</div>
<div class="pager" role="navigation" aria-label="related navigation">
- <a rel="prev" title="Building the image" href="build.html" >
+ <a rel="prev" title="airflow.executors.executor_constants" href="../executor_constants/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="p">Previous</button>
</a>
- <a rel="next" title="Recipes" href="recipes.html" >
+ <a rel="next" title="airflow.executors.kubernetes_executor" href="../kubernetes_executor/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="n">Next</button>
</a>
</div>
@@ -857,12 +884,32 @@ To disable check, set <code class="docutils literal notranslate"><span class="pr
<nav class="wy-nav-side-toc">
<div class="wy-menu-vertical">
<ul>
-<li><a class="reference internal" href="#">Entrypoint</a><ul>
-<li><a class="reference internal" href="#allowing-arbitrary-user-to-run-the-container">Allowing arbitrary user to run the container</a></li>
-<li><a class="reference internal" href="#waits-for-airflow-db-connection">Waits for Airflow DB connection</a></li>
-<li><a class="reference internal" href="#upgrading-airflow-db">Upgrading Airflow DB</a></li>
-<li><a class="reference internal" href="#creating-admin-user">Creating admin user</a></li>
-<li><a class="reference internal" href="#waits-for-celery-broker-connection">Waits for celery broker connection</a></li>
+<li><a class="reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a><ul>
+<li><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li><a class="reference internal" href="#airflow.executors.executor_loader.log">log</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader">ExecutorLoader</a><ul>
+<li><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader._default_executor">_default_executor</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader.executors">executors</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader.get_default_executor">get_default_executor</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader.load_executor">load_executor</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.executor_loader.ExecutorLoader.__load_celery_kubernetes_executor">__load_celery_kubernetes_executor</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.executor_loader.UNPICKLEABLE_EXECUTORS">UNPICKLEABLE_EXECUTORS</a></li>
+</ul>
+</li>
</ul>
</li>
</ul>
@@ -887,7 +934,7 @@ To disable check, set <code class="docutils literal notranslate"><span class="pr
<div class="base-layout--button">
- <a href="https://github.com/apache/airflow/edit/devel/docs/docker-stack/entrypoint.rst" rel="nofollow">
+ <a href="https://github.com/apache/airflow/edit/devel/docs/apache-airflow/_api/airflow/executors/executor_loader/index.rst" rel="nofollow">
<button class="btn-hollow btn-brown btn-with-icon with-box-shadow button-fixed">
<svg xmlns="http://www.w3.org/2000/svg" width="30.76" height="30">
@@ -1021,15 +1068,16 @@ To disable check, set <code class="docutils literal notranslate"><span class="pr
</span>
</div>
</footer>
- <script type="text/javascript" src="_static/_gen/js/docs.js"></script>
- <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../../_static/_gen/js/docs.js"></script>
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
<script src="https://code.jquery.com/jquery-3.4.1.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script>
- <script src="_static/jquery.js"></script>
- <script src="_static/underscore.js"></script>
- <script src="_static/doctools.js"></script>
- <script src="_static/clipboard.min.js"></script>
- <script src="_static/copybutton.js"></script>
+ <script src="../../../../_static/jquery.js"></script>
+ <script src="../../../../_static/underscore.js"></script>
+ <script src="../../../../_static/doctools.js"></script>
+ <script src="../../../../_static/clipboard.min.js"></script>
+ <script src="../../../../_static/copybutton.js"></script>
+ <script src="../../../../_static/jira-links.js"></script>
</body>
</html>
\ No newline at end of file
diff --git a/docs-archive/docker-stack/index.html b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/index.html
similarity index 78%
copy from docs-archive/docker-stack/index.html
copy to docs-archive/apache-airflow/2.1.0/_api/airflow/executors/index.html
index d012095..30c2b9f 100644
--- a/docs-archive/docker-stack/index.html
+++ b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/index.html
@@ -9,14 +9,16 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Docker Image for Apache Airflow — docker-stack Documentation</title>
- <link rel="stylesheet" href="_static/_gen/css/main.min.css" type="text/css" />
- <link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
- <link rel="stylesheet" type="text/css" href="_static/_gen/css/main-custom.min.css" />
- <link rel="shortcut icon" href="_static/pin_32.png"/>
- <link rel="index" title="Index" href="genindex.html" />
- <link rel="search" title="Search" href="search.html" />
- <link rel="next" title="Building the image" href="build.html" />
+ <title>airflow.executors — Airflow Documentation</title>
+ <link rel="stylesheet" href="../../../_static/_gen/css/main.min.css" type="text/css" />
+ <link rel="stylesheet" type="text/css" href="../../../_static/copybutton.css" />
+ <link rel="stylesheet" type="text/css" href="../../../_static/_gen/css/main-custom.min.css" />
+ <link rel="stylesheet" type="text/css" href="../../../_static/graphviz.css" />
+ <link rel="shortcut icon" href="../../../_static/pin_32.png"/>
+ <link rel="index" title="Index" href="../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../search.html" />
+ <link rel="next" title="airflow.executors.base_executor" href="base_executor/index.html" />
+ <link rel="prev" title="airflow.hooks.zendesk_hook" href="../hooks/zendesk_hook/index.html" />
<script type="application/javascript">
var doNotTrack = false;
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
@@ -25,9 +27,9 @@
</script>
<script async src='https://www.google-analytics.com/analytics.js'></script>
-
+
</head><body class="td-section">
-
+
<header>
@@ -67,38 +69,38 @@
<div class="navbar__menu-content" id="main_navbar">
<div class="navbar__links-container">
-
+
<a class="navbar__text-link" href="/community/">
Community
</a>
-
+
<a class="navbar__text-link" href="/meetups/">
Meetups
</a>
-
+
<a class="navbar__text-link" href="/docs/">
Documentation
</a>
-
+
<a class="navbar__text-link" href="/use-cases/">
Use-cases
</a>
-
+
<a class="navbar__text-link" href="/announcements/">
Announcements
</a>
-
+
<a class="navbar__text-link" href="/blog/">
Blog
</a>
-
+
<a class="navbar__text-link" href="/ecosystem/">
Ecosystem
</a>
-
+
</div>
-
+
</div>
@@ -142,38 +144,38 @@
<div class="navbar__menu-content" id="main_navbar">
<div class="navbar__links-container">
-
+
<a class="navbar__text-link" href="/community/">
Community
</a>
-
+
<a class="navbar__text-link" href="/meetups/">
Meetups
</a>
-
+
<a class="navbar__text-link" href="/docs/">
Documentation
</a>
-
+
<a class="navbar__text-link" href="/use-cases/">
Use-cases
</a>
-
+
<a class="navbar__text-link" href="/announcements/">
Announcements
</a>
-
+
<a class="navbar__text-link" href="/blog/">
Blog
</a>
-
+
<a class="navbar__text-link" href="/ecosystem/">
Ecosystem
</a>
-
+
</div>
-
+
</div>
</div>
@@ -184,8 +186,8 @@
<div class="roadmap container-fluid td-default base-layout">
-
-
+
+
<div class="content-drawer-wrapper">
<button class="content-drawer__toggle-button" id="content-open-button">
@@ -271,7 +273,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -282,7 +284,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -312,15 +314,79 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1"><a class="reference internal" href="recipes.html">Recipes</a></li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
<p class="caption"><span class="caption-text">References</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul>
+<li class="toctree-l5"><a class="reference internal" href="#submodules">Submodules</a><ul>
+<li class="toctree-l6"><a class="reference internal" href="base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -395,16 +461,16 @@
</div>
</div>
</div>
-
+
<div class="d-flex">
-
-
+
+
<div class="td-sidebar desktop-only d-print-none">
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -415,7 +481,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -445,15 +511,79 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Home</a></li>
-<li class="toctree-l1"><a class="reference internal" href="build.html">Building the image</a></li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1"><a class="reference internal" href="recipes.html">Recipes</a></li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
<p class="caption"><span class="caption-text">References</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul>
+<li class="toctree-l5"><a class="reference internal" href="#submodules">Submodules</a><ul>
+<li class="toctree-l6"><a class="reference internal" href="base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -525,15 +655,15 @@
</style>
</div>
+
-
-
+
<main class="col-12 col-md-9 col-xl-8" role="main">
+
-
-
+
@@ -541,48 +671,51 @@
<div role="navigation" aria-label="breadcrumbs navigation" class="d-none d-md-block d-print-none">
<ul class="breadcrumb">
-
- <li class="breadcrumb-item"><a href="#" class="icon icon-home"> Home</a></li>
-
- <li class="breadcrumb-item"><a href="index.html"> Docker Image for Apache Airflow</a></li>
-
+
+ <li class="breadcrumb-item"><a href="../../../index.html" class="icon icon-home"> Home</a></li>
+
+ <li class="breadcrumb-item"><a href="../../../python-api-ref.html">Python API Reference</a></li>
+
+ <li class="breadcrumb-item"><a href="index.html"> <code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a></li>
+
</ul>
</div>
-
+
<div class="rst-content">
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
-
- <blockquote>
-<div></div></blockquote>
-<a class="reference internal image-reference" href="_images/docker-logo.png"><img alt="_images/docker-logo.png" src="_images/docker-logo.png" style="width: 100px;" /></a>
-<div class="section" id="docker-image-for-apache-airflow">
-<h1>Docker Image for Apache Airflow<a class="headerlink" href="#docker-image-for-apache-airflow" title="Permalink to this headline">¶</a></h1>
+
+ <div class="section" id="module-airflow.executors">
+<span id="airflow-executors"></span><h1><a class="reference internal" href="#module-airflow.executors" title="airflow.executors"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><a class="headerlink" href="#module-airflow.executors" title="Permalink to this headline">¶</a></h1>
+<p><p>Executors.</p>
+</p>
+<div class="section" id="submodules">
+<h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this headline">¶</a></h2>
<div class="toctree-wrapper compound">
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l1"><a class="reference internal" href="celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l1"><a class="reference internal" href="celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l1"><a class="reference internal" href="dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l1"><a class="reference internal" href="debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l1"><a class="reference internal" href="executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l1"><a class="reference internal" href="executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l1"><a class="reference internal" href="kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
+<li class="toctree-l1"><a class="reference internal" href="local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l1"><a class="reference internal" href="sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
</div>
-<div class="toctree-wrapper compound">
</div>
-<p>For the ease of deployment in production, the community releases a production-ready reference container
-image.</p>
-<p>The docker image provided (as convenience binary package) in the
-<a class="reference external" href="https://hub.docker.com/r/apache/airflow">apache/airflow DockerHub</a> is a bare image
-that has a few external dependencies and extras installed..</p>
-<p>The Apache Airflow image provided as convenience package is optimized for size, so
-it provides just a bare minimal set of the extras and dependencies installed and in most cases
-you want to either extend or customize the image. You can see all possible extras in
-<a class="reference external" href="/docs/apache-airflow/stable/extra-packages-ref.html" title="(in apache-airflow v2.1.0.dev0)"><span>Reference for package extras</span></a>. The set of extras used in Airflow Production image are available in the
-<a class="reference external" href="https://github.com/apache/airflow/blob/2c6c7fdb2308de98e142618836bdf414df9768c8/Dockerfile#L39">Dockerfile</a>.</p>
-<p>The production images are build in DockerHub from released version and release candidates. There
-are also images published from branches but they are used mainly for development and testing purpose.
-See <a class="reference external" href="https://github.com/apache/airflow/blob/master/CONTRIBUTING.rst#airflow-git-branches">Airflow Git Branching</a>
-for details.</p>
</div>
- <div class="pager" role="navigation" aria-label="related navigation"><button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" disabled>Previous</button>
- <a rel="next" title="Building the image" href="build.html" >
+ <div class="pager" role="navigation" aria-label="related navigation">
+ <a rel="prev" title="airflow.hooks.zendesk_hook" href="../hooks/zendesk_hook/index.html" >
+ <button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="p">Previous</button>
+ </a>
+ <a rel="next" title="airflow.executors.base_executor" href="base_executor/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="n">Next</button>
</a>
</div>
@@ -643,26 +776,29 @@ for details.</p>
</div>
</div>
-
+
</main>
+
-
-
-
+
+
<nav class="wy-nav-side-toc">
<div class="wy-menu-vertical">
<ul>
-<li><a class="reference internal" href="#">Docker Image for Apache Airflow</a><ul>
+<li><a class="reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul>
+<li><a class="reference internal" href="#submodules">Submodules</a></li>
</ul>
</li>
</ul>
</div>
</nav>
-
+
</div>
+
+
@@ -670,14 +806,12 @@ for details.</p>
-
-
-
-
-
-
+
+
+
+
<div class="base-layout--button">
- <a href="https://github.com/apache/airflow/edit/devel/docs/docker-stack/index.rst" rel="nofollow">
+ <a href="https://github.com/apache/airflow/edit/devel/docs/apache-airflow/_api/airflow/executors/index.rst" rel="nofollow">
<button class="btn-hollow btn-brown btn-with-icon with-box-shadow button-fixed">
<svg xmlns="http://www.w3.org/2000/svg" width="30.76" height="30">
@@ -688,12 +822,12 @@ for details.</p>
</button>
</a>
</div>
-
+
</div>
-
+
<footer>
<div class="footer-section footer-section__media-section">
@@ -768,7 +902,7 @@ for details.</p>
</a>
</div>
-
+
<div class="footer-section__media-section--button-with-text">
<span class="footer-section__media-section--text">Want to be a part of Apache Airflow?</span>
@@ -778,7 +912,7 @@ for details.</p>
</a>
</div>
-
+
</div>
<div class="footer-section footer-section__policies-section">
@@ -801,7 +935,7 @@ for details.</p>
<a href="https://www.apache.org/security/" class="footer-section__policies-section--policy-item">
<span>Security</span>
</a>
-
+
</div>
</div>
@@ -811,15 +945,16 @@ for details.</p>
</span>
</div>
</footer>
- <script type="text/javascript" src="_static/_gen/js/docs.js"></script>
- <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
+ <script type="text/javascript" src="../../../_static/_gen/js/docs.js"></script>
+ <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
<script src="https://code.jquery.com/jquery-3.4.1.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script>
- <script src="_static/jquery.js"></script>
- <script src="_static/underscore.js"></script>
- <script src="_static/doctools.js"></script>
- <script src="_static/clipboard.min.js"></script>
- <script src="_static/copybutton.js"></script>
+ <script src="../../../_static/jquery.js"></script>
+ <script src="../../../_static/underscore.js"></script>
+ <script src="../../../_static/doctools.js"></script>
+ <script src="../../../_static/clipboard.min.js"></script>
+ <script src="../../../_static/copybutton.js"></script>
+ <script src="../../../_static/jira-links.js"></script>
</body>
-</html>
+</html>
\ No newline at end of file
diff --git a/docs-archive/docker-stack/build.html b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/kubernetes_executor/index.html
similarity index 52%
copy from docs-archive/docker-stack/build.html
copy to docs-archive/apache-airflow/2.1.0/_api/airflow/executors/kubernetes_executor/index.html
index ef84de3..83804e0 100644
--- a/docs-archive/docker-stack/build.html
+++ b/docs-archive/apache-airflow/2.1.0/_api/airflow/executors/kubernetes_executor/index.html
@@ -9,15 +9,16 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
- <title>Building the image — docker-stack Documentation</title>
- <link rel="stylesheet" href="_static/_gen/css/main.min.css" type="text/css" />
- <link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
- <link rel="stylesheet" type="text/css" href="_static/_gen/css/main-custom.min.css" />
- <link rel="shortcut icon" href="_static/pin_32.png"/>
- <link rel="index" title="Index" href="genindex.html" />
- <link rel="search" title="Search" href="search.html" />
- <link rel="next" title="Entrypoint" href="entrypoint.html" />
- <link rel="prev" title="Docker Image for Apache Airflow" href="index.html" />
+ <title>airflow.executors.kubernetes_executor — Airflow Documentation</title>
+ <link rel="stylesheet" href="../../../../_static/_gen/css/main.min.css" type="text/css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/copybutton.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/_gen/css/main-custom.min.css" />
+ <link rel="stylesheet" type="text/css" href="../../../../_static/graphviz.css" />
+ <link rel="shortcut icon" href="../../../../_static/pin_32.png"/>
+ <link rel="index" title="Index" href="../../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../../search.html" />
+ <link rel="next" title="airflow.executors.local_executor" href="../local_executor/index.html" />
+ <link rel="prev" title="airflow.executors.executor_loader" href="../executor_loader/index.html" />
<script type="application/javascript">
var doNotTrack = false;
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
@@ -272,7 +273,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -283,7 +284,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -313,42 +314,193 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Building the image</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#extending-vs-customizing-the-image">Extending vs. customizing the image</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#extending-the-image">Extending the image</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#examples-of-image-extending">Examples of image extending</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="#an-apt-package-example">An <code class="docutils literal notranslate"><span class="pre">apt</span></code> package example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#a-pypi-package-example">A <code class="docutils literal notranslate"><span class="pre">PyPI</span></code> package example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#a-umask-requiring-example">A <code class="docutils literal notranslate"><span class="pre">umask</span></code> requiring example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#a-build-essential-requiring-package-example">A <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> requiring package example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#dag-embedding-example">DAG embedding example</a></li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
-</li>
-<li class="toctree-l2"><a class="reference internal" href="#customizing-the-image">Customizing the image</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#examples-of-image-customizing">Examples of image customizing</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="#building-from-pypi-packages">Building from PyPI packages</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#building-optimized-images">Building optimized images</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#building-from-github">Building from GitHub</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#using-custom-installation-sources">Using custom installation sources</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#build-images-in-security-restricted-environments">Build images in security restricted environments</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#modifying-the-dockerfile">Modifying the Dockerfile</a></li>
+<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6"><a class="reference internal" href="../base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobType">KubernetesJobType</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesResultsType">KubernetesResultsType</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesWatchType">KubernetesWatchType</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.ResourceVersion">ResourceVersion</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.ResourceVersion._instance">_instance</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.ResourceVersion.resource_version">resource_version</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.ResourceVersion.__new__">__new__</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="#more-details">More details</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="#build-args-reference">Build Args reference</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#the-architecture-of-the-images">The architecture of the images</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobWatcher">KubernetesJobWatcher</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobWatcher.run">run</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobWatcher._run">_run</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobWatcher.process_error">process_error</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobWatcher.process_status">process_status</a></li>
</ul>
</li>
</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler">AirflowKubernetesScheduler</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.run_pod_async">run_pod_async</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler._make_kube_watcher">_make_kube_watcher</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler._health_check_kube_watcher">_health_check_kube_watcher</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.run_next">run_next</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.delete_pod">delete_pod</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.sync">sync</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.process_watcher_task">process_watcher_task</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler._flush_watcher_queue">_flush_watcher_queue</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.terminate">terminate</a></li>
+</ul>
</li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1"><a class="reference internal" href="recipes.html">Recipes</a></li>
</ul>
-<p class="caption"><span class="caption-text">References</span></p>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.get_base_pod_from_template">get_base_pod_from_template</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor">KubernetesExecutor</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.clear_not_launched_queued_tasks">clear_not_launched_queued_tasks</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.start">start</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.execute_async">execute_async</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.sync">sync</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor._check_worker_pods_pending_timeout">_check_worker_pods_pending_timeout</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor._change_state">_change_state</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.try_adopt_task_instances">try_adopt_task_instances</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.adopt_launched_task">adopt_launched_task</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor._adopt_completed_pods">_adopt_completed_pods</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor._flush_task_queue">_flush_task_queue</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor._flush_result_queue">_flush_result_queue</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.end">end</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.terminate">terminate</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -432,7 +584,7 @@
<div id="docs-version-selector" class="docs-version-selector sidebar__version-selector">
<a class="dropdown-toggle" href="#" id="versionDropdown" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
- <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">devel</span>
+ <span class="bodytext__medium--greyish-brown">Version: </span><span class="version">2.1.0</span>
</a>
<div class="dropdown-menu" aria-labelledby="navbarDropdownMenuLink">
@@ -443,7 +595,7 @@
</div>
<div class="searchb-box">
- <form class="search-form" action="search.html" method="get">
+ <form class="search-form" action="../../../../search.html" method="get">
<input class="search-form__input" type="text" name="q" placeholder="Search docs" size="16">
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
@@ -473,42 +625,193 @@
<div class="toctree" role="navigation" aria-label="main navigation">
- <ul class="current">
-<li class="toctree-l1"><a class="reference internal" href="index.html">Home</a></li>
-<li class="toctree-l1 current"><a class="current reference internal" href="#">Building the image</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="#extending-vs-customizing-the-image">Extending vs. customizing the image</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#extending-the-image">Extending the image</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#examples-of-image-extending">Examples of image extending</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="#an-apt-package-example">An <code class="docutils literal notranslate"><span class="pre">apt</span></code> package example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#a-pypi-package-example">A <code class="docutils literal notranslate"><span class="pre">PyPI</span></code> package example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#a-umask-requiring-example">A <code class="docutils literal notranslate"><span class="pre">umask</span></code> requiring example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#a-build-essential-requiring-package-example">A <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> requiring package example</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#dag-embedding-example">DAG embedding example</a></li>
+ <p class="caption"><span class="caption-text">Content</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../index.html">Home</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../start/index.html">Quick start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrading-to-2.html">Upgrading to Airflow 2.0+</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../upgrade-check.html">Upgrade Check Script</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../tutorial_taskflow_api.html">Tutorial on the Taskflow API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../concepts/index.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../executor/index.html">Executor</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-run.html">DAG Runs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../security/index.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../logging-monitoring/index.html">Logging & Monitoring</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../usage-cli.html">Using the CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../kubernetes.html">Kubernetes</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../dag-serialization.html">DAG Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../modules_management.html">Modules Management</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../release-process.html">Release policies</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../changelog.html">Changelog</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../best-practices.html">Best Practices</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../production-deployment.html">Production Deployment</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../backport-providers.html">Backport Providers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../privacy_notice.html">Privacy Notice</a></li>
</ul>
-</li>
-<li class="toctree-l2"><a class="reference internal" href="#customizing-the-image">Customizing the image</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#examples-of-image-customizing">Examples of image customizing</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="#building-from-pypi-packages">Building from PyPI packages</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#building-optimized-images">Building optimized images</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#building-from-github">Building from GitHub</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#using-custom-installation-sources">Using custom installation sources</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#build-images-in-security-restricted-environments">Build images in security restricted environments</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#modifying-the-dockerfile">Modifying the Dockerfile</a></li>
+<p class="caption"><span class="caption-text">References</span></p>
+<ul class="current">
+<li class="toctree-l1"><a class="reference internal" href="../../../../operators-and-hooks-ref.html">Operators and hooks</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../cli-and-env-variables-ref.html">CLI</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../macros-ref.html">Macros</a></li>
+<li class="toctree-l1 current"><a class="reference internal" href="../../../../python-api-ref.html">Python API</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#operators">Operators</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#hooks">Hooks</a></li>
+<li class="toctree-l2 current"><a class="reference internal" href="../../../../python-api-ref.html#executors">Executors</a><ul class="current">
+<li class="toctree-l3 current"><a class="reference internal" href="../../../../python-api-ref.html#executors-packages">Executors packages</a><ul class="current">
+<li class="toctree-l4 current"><a class="reference internal" href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a><ul class="current">
+<li class="toctree-l5 current"><a class="reference internal" href="../index.html#submodules">Submodules</a><ul class="current">
+<li class="toctree-l6"><a class="reference internal" href="../base_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.base_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../celery_kubernetes_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.celery_kubernetes_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../dask_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.dask_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../debug_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.debug_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_constants/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_constants</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../executor_loader/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.executor_loader</span></code></a></li>
+<li class="toctree-l6 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a><ul>
+<li class="toctree-l7"><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobType">KubernetesJobType</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesResultsType">KubernetesResultsType</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesWatchType">KubernetesWatchType</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.ResourceVersion">ResourceVersion</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.ResourceVersion._instance">_instance</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.ResourceVersion.resource_version">resource_version</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.ResourceVersion.__new__">__new__</a></li>
</ul>
</li>
-<li class="toctree-l2"><a class="reference internal" href="#more-details">More details</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="#build-args-reference">Build Args reference</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#the-architecture-of-the-images">The architecture of the images</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobWatcher">KubernetesJobWatcher</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobWatcher.run">run</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobWatcher._run">_run</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobWatcher.process_error">process_error</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobWatcher.process_status">process_status</a></li>
</ul>
</li>
</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler">AirflowKubernetesScheduler</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.run_pod_async">run_pod_async</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler._make_kube_watcher">_make_kube_watcher</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler._health_check_kube_watcher">_health_check_kube_watcher</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.run_next">run_next</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.delete_pod">delete_pod</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.sync">sync</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.process_watcher_task">process_watcher_task</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler._flush_watcher_queue">_flush_watcher_queue</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.terminate">terminate</a></li>
+</ul>
</li>
-<li class="toctree-l1"><a class="reference internal" href="entrypoint.html">Entrypoint</a></li>
-<li class="toctree-l1"><a class="reference internal" href="recipes.html">Recipes</a></li>
</ul>
-<p class="caption"><span class="caption-text">References</span></p>
<ul>
-<li class="toctree-l1"><a class="reference internal" href="build-arg-ref.html">Image build arguments reference</a></li>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.get_base_pod_from_template">get_base_pod_from_template</a></li>
+</ul>
+<ul>
+<li class="toctree-l8"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor">KubernetesExecutor</a><ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.clear_not_launched_queued_tasks">clear_not_launched_queued_tasks</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.start">start</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.execute_async">execute_async</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.sync">sync</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor._check_worker_pods_pending_timeout">_check_worker_pods_pending_timeout</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor._change_state">_change_state</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.try_adopt_task_instances">try_adopt_task_instances</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.adopt_launched_task">adopt_launched_task</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor._adopt_completed_pods">_adopt_completed_pods</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor._flush_task_queue">_flush_task_queue</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor._flush_result_queue">_flush_result_queue</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.end">end</a></li>
+</ul>
+<ul>
+<li class="toctree-l9"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.terminate">terminate</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l6"><a class="reference internal" href="../local_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.local_executor</span></code></a></li>
+<li class="toctree-l6"><a class="reference internal" href="../sequential_executor/index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.sequential_executor</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#models">Models</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#exceptions">Exceptions</a></li>
+<li class="toctree-l2"><a class="reference internal" href="../../../../python-api-ref.html#secrets-backends">Secrets Backends</a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../stable-rest-api-ref.html">Stable REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../deprecated-rest-api-ref.html">Deprecated REST API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../configurations-ref.html">Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../extra-packages-ref.html">Extra packages</a></li>
</ul>
</div>
@@ -597,9 +900,13 @@
<ul class="breadcrumb">
- <li class="breadcrumb-item"><a href="index.html" class="icon icon-home"> Home</a></li>
+ <li class="breadcrumb-item"><a href="../../../../index.html" class="icon icon-home"> Home</a></li>
+
+ <li class="breadcrumb-item"><a href="../../../../python-api-ref.html">Python API Reference</a></li>
+
+ <li class="breadcrumb-item"><a href="../index.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors</span></code></a></li>
- <li class="breadcrumb-item"><a href="build.html"> Building the image</a></li>
+ <li class="breadcrumb-item"><a href="index.html"> <code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a></li>
</ul>
</div>
@@ -610,627 +917,285 @@
<div class="bodywrapper">
<div class="body" role="main">
- <blockquote>
-<div></div></blockquote>
-<div class="section" id="building-the-image">
-<h1>Building the image<a class="headerlink" href="#building-the-image" title="Permalink to this headline">¶</a></h1>
-<p>Before you dive-deeply in the way how the Airflow Image is build, named and why we are doing it the
-way we do, you might want to know very quickly how you can extend or customize the existing image
-for Apache Airflow. This chapter gives you a short answer to those questions.</p>
-<div class="section" id="extending-vs-customizing-the-image">
-<h2>Extending vs. customizing the image<a class="headerlink" href="#extending-vs-customizing-the-image" title="Permalink to this headline">¶</a></h2>
-<p>Here is the comparison of the two types of building images. Here is your guide if you want to choose
-how you want to build your image.</p>
-<table class="docutils align-default">
-<colgroup>
-<col style="width: 68%" />
-<col style="width: 14%" />
-<col style="width: 17%" />
-</colgroup>
-<thead>
-<tr class="row-odd"><th class="head"></th>
-<th class="head"><p>Extending</p></th>
-<th class="head"><p>Customizing</p></th>
-</tr>
-</thead>
-<tbody>
-<tr class="row-even"><td><p>Can be built without airflow sources</p></td>
-<td><p>Yes</p></td>
-<td><p>No</p></td>
-</tr>
-<tr class="row-odd"><td><p>Uses familiar 'FROM ' pattern of image building</p></td>
-<td><p>Yes</p></td>
-<td><p>No</p></td>
-</tr>
-<tr class="row-even"><td><p>Requires only basic knowledge about images</p></td>
-<td><p>Yes</p></td>
-<td><p>No</p></td>
-</tr>
-<tr class="row-odd"><td><p>Builds quickly</p></td>
-<td><p>Yes</p></td>
-<td><p>No</p></td>
-</tr>
-<tr class="row-even"><td><p>Produces image heavily optimized for size</p></td>
-<td><p>No</p></td>
-<td><p>Yes</p></td>
-</tr>
-<tr class="row-odd"><td><p>Can build from custom airflow sources (forks)</p></td>
-<td><p>No</p></td>
-<td><p>Yes</p></td>
-</tr>
-<tr class="row-even"><td><p>Can build on air-gaped system</p></td>
-<td><p>No</p></td>
-<td><p>Yes</p></td>
-</tr>
-</tbody>
-</table>
-<p>TL;DR; If you have a need to build custom image, it is easier to start with "Extending" however if your
-dependencies require compilation step or when your require to build the image from security vetted
-packages, switching to "Customizing" the image provides much more optimized images. In the example further
-where we compare equivalent "Extending" and "Customizing" the image, similar images build by
-Extending vs. Customization had shown 1.1GB vs 874MB image sizes respectively - with 20% improvement in
-size of the Customized image.</p>
-<div class="admonition note">
-<p class="admonition-title">Note</p>
-<p>You can also combine both - customizing & extending the image in one. You can build your
-optimized base image first using <code class="docutils literal notranslate"><span class="pre">customization</span></code> method (for example by your admin team) with all
-the heavy compilation required dependencies and you can publish it in your registry and let others
-<code class="docutils literal notranslate"><span class="pre">extend</span></code> your image using <code class="docutils literal notranslate"><span class="pre">FROM</span></code> and add their own lightweight dependencies. This reflects well
-the split where typically "Casual" users will Extend the image and "Power-users" will customize it.</p>
-</div>
-<p>Airflow Summit 2020's <a class="reference external" href="https://youtu.be/wDr3Y7q2XoI">Production Docker Image</a> talk provides more
-details about the context, architecture and customization/extension methods for the Production Image.</p>
-</div>
-<div class="section" id="extending-the-image">
-<h2>Extending the image<a class="headerlink" href="#extending-the-image" title="Permalink to this headline">¶</a></h2>
-<p>Extending the image is easiest if you just need to add some dependencies that do not require
-compiling. The compilation framework of Linux (so called <code class="docutils literal notranslate"><span class="pre">build-essential</span></code>) is pretty big, and
-for the production images, size is really important factor to optimize for, so our Production Image
-does not contain <code class="docutils literal notranslate"><span class="pre">build-essential</span></code>. If you need compiler like gcc or g++ or make/cmake etc. - those
-are not found in the image and it is recommended that you follow the "customize" route instead.</p>
-<p>How to extend the image - it is something you are most likely familiar with - simply
-build a new image using Dockerfile's <code class="docutils literal notranslate"><span class="pre">FROM</span></code> directive and add whatever you need. Then you can add your
-Debian dependencies with <code class="docutils literal notranslate"><span class="pre">apt</span></code> or PyPI dependencies with <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span></code> or any other stuff you need.</p>
-<p>You should be aware, about a few things:</p>
-<ul class="simple">
-<li><p>The production image of airflow uses "airflow" user, so if you want to add some of the tools
-as <code class="docutils literal notranslate"><span class="pre">root</span></code> user, you need to switch to it with <code class="docutils literal notranslate"><span class="pre">USER</span></code> directive of the Dockerfile and switch back to
-<code class="docutils literal notranslate"><span class="pre">airflow</span></code> user when you are done. Also you should remember about following the
-<a class="reference external" href="https://docs.docker.com/develop/develop-images/dockerfile_best-practices/">best practises of Dockerfiles</a>
-to make sure your image is lean and small.</p></li>
-<li><p>The PyPI dependencies in Apache Airflow are installed in the user library, of the "airflow" user, so
-PIP packages are installed to <code class="docutils literal notranslate"><span class="pre">~/.local</span></code> folder as if the <code class="docutils literal notranslate"><span class="pre">--user</span></code> flag was specified when running PIP.
-Note also that using <code class="docutils literal notranslate"><span class="pre">--no-cache-dir</span></code> is a good idea that can help to make your image smaller.</p></li>
-</ul>
-<div class="admonition note">
-<p class="admonition-title">Note</p>
-<p>Only as of <code class="docutils literal notranslate"><span class="pre">2.0.1</span></code> image the <code class="docutils literal notranslate"><span class="pre">--user</span></code> flag is turned on by default by setting <code class="docutils literal notranslate"><span class="pre">PIP_USER</span></code> environment
-variable to <code class="docutils literal notranslate"><span class="pre">true</span></code>. This can be disabled by un-setting the variable or by setting it to <code class="docutils literal notranslate"><span class="pre">false</span></code>. In the
-2.0.0 image you had to add the <code class="docutils literal notranslate"><span class="pre">--user</span></code> flag as <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--user</span></code> command.</p>
-</div>
-<ul class="simple">
-<li><p>If your apt, or PyPI dependencies require some of the <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> or other packages that need
-to compile your python dependencies, then your best choice is to follow the "Customize the image" route,
-because you can build a highly-optimized (for size) image this way. However it requires to checkout sources
-of Apache Airflow, so you might still want to choose to add <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> to your image,
-even if your image will be significantly bigger.</p></li>
-<li><p>You can also embed your dags in the image by simply adding them with COPY directive of Airflow.
-The DAGs in production image are in <code class="docutils literal notranslate"><span class="pre">/opt/airflow/dags</span></code> folder.</p></li>
-<li><p>You can build your image without any need for Airflow sources. It is enough that you place the
-<code class="docutils literal notranslate"><span class="pre">Dockerfile</span></code> and any files that are referred to (such as Dag files) in a separate directory and run
-a command <code class="docutils literal notranslate"><span class="pre">docker</span> <span class="pre">build</span> <span class="pre">.</span> <span class="pre">--tag</span> <span class="pre">my-image:my-tag</span></code> (where <code class="docutils literal notranslate"><span class="pre">my-image</span></code> is the name you want to name it
-and <code class="docutils literal notranslate"><span class="pre">my-tag</span></code> is the tag you want to tag the image with.</p></li>
-<li><p>If your way of extending image requires to create writable directories, you MUST remember about adding
-<code class="docutils literal notranslate"><span class="pre">umask</span> <span class="pre">0002</span></code> step in your RUN command. This is necessary in order to accommodate our approach for
-running the image with an arbitrary user. Such user will always run with <code class="docutils literal notranslate"><span class="pre">GID=0</span></code> -
-the entrypoint will prevent non-root GIDs. You can read more about it in
-<a class="reference internal" href="entrypoint.html#arbitrary-docker-user"><span class="std std-ref">arbitrary docker user</span></a> documentation for the entrypoint. The
-<code class="docutils literal notranslate"><span class="pre">umask</span> <span class="pre">0002</span></code> is set as default when you enter the image, so any directories you create by default
-in runtime, will have <code class="docutils literal notranslate"><span class="pre">GID=0</span></code> and will be group-writable.</p></li>
-</ul>
-<div class="admonition note">
-<p class="admonition-title">Note</p>
-<p>Only as of <code class="docutils literal notranslate"><span class="pre">2.0.2</span></code> the default group of <code class="docutils literal notranslate"><span class="pre">airflow</span></code> user is <code class="docutils literal notranslate"><span class="pre">root</span></code>. Previously it was <code class="docutils literal notranslate"><span class="pre">airflow</span></code>,
-so if you are building your images based on an earlier image, you need to manually change the default
-group for airflow user:</p>
-</div>
-<div class="highlight-docker notranslate"><div class="highlight"><pre><span></span><span class="k">RUN</span> usermod -g <span class="m">0</span> airflow
-</pre></div>
-</div>
-</div>
-<div class="section" id="examples-of-image-extending">
-<h2>Examples of image extending<a class="headerlink" href="#examples-of-image-extending" title="Permalink to this headline">¶</a></h2>
-<div class="section" id="an-apt-package-example">
-<h3>An <code class="docutils literal notranslate"><span class="pre">apt</span></code> package example<a class="headerlink" href="#an-apt-package-example" title="Permalink to this headline">¶</a></h3>
-<p>The following example adds <code class="docutils literal notranslate"><span class="pre">vim</span></code> to the airflow image.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/extending/add-apt-packages/Dockerfile</span></p>
-<div class="highlight-Dockerfile notranslate"><div class="highlight"><pre><span></span><span class="k">FROM</span> <span class="s">apache/airflow:2.0.2</span>
-<span class="k">USER</span><span class="s"> root</span>
-<span class="k">RUN</span> apt-get update <span class="se">\</span>
- <span class="o">&&</span> apt-get install -y --no-install-recommends <span class="se">\</span>
- vim <span class="se">\</span>
- <span class="o">&&</span> apt-get autoremove -yqq --purge <span class="se">\</span>
- <span class="o">&&</span> apt-get clean <span class="se">\</span>
- <span class="o">&&</span> rm -rf /var/lib/apt/lists/*
-<span class="k">USER</span><span class="s"> airflow</span>
-</pre></div>
-</div>
-</div>
-</div>
-<div class="section" id="a-pypi-package-example">
-<h3>A <code class="docutils literal notranslate"><span class="pre">PyPI</span></code> package example<a class="headerlink" href="#a-pypi-package-example" title="Permalink to this headline">¶</a></h3>
-<p>The following example adds <code class="docutils literal notranslate"><span class="pre">lxml</span></code> python package from PyPI to the image.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/extending/add-pypi-packages/Dockerfile</span></p>
-<div class="highlight-Dockerfile notranslate"><div class="highlight"><pre><span></span><span class="k">FROM</span> <span class="s">apache/airflow:2.0.2</span>
-<span class="k">RUN</span> pip install --no-cache-dir lxml
-</pre></div>
-</div>
-</div>
-</div>
-<div class="section" id="a-umask-requiring-example">
-<h3>A <code class="docutils literal notranslate"><span class="pre">umask</span></code> requiring example<a class="headerlink" href="#a-umask-requiring-example" title="Permalink to this headline">¶</a></h3>
-<p>The following example adds a new directory that is supposed to be writable for any arbitrary user
-running the container.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/extending/writable-directory/Dockerfile</span></p>
-<div class="highlight-Dockerfile notranslate"><div class="highlight"><pre><span></span><span class="k">FROM</span> <span class="s">apache/airflow:2.0.2</span>
-<span class="k">RUN</span> <span class="nb">umask</span> <span class="m">0002</span><span class="p">;</span> <span class="se">\</span>
- mkdir -p ~/writeable-directory
-</pre></div>
-</div>
-</div>
-</div>
-<div class="section" id="a-build-essential-requiring-package-example">
-<h3>A <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> requiring package example<a class="headerlink" href="#a-build-essential-requiring-package-example" title="Permalink to this headline">¶</a></h3>
-<p>The following example adds <code class="docutils literal notranslate"><span class="pre">mpi4py</span></code> package which requires both <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> and <code class="docutils literal notranslate"><span class="pre">mpi</span> <span class="pre">compiler</span></code>.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/extending/add-build-essential-extend/Dockerfile</span></p>
-<div class="highlight-Dockerfile notranslate"><div class="highlight"><pre><span></span><span class="k">FROM</span> <span class="s">apache/airflow:2.0.2</span>
-<span class="k">USER</span><span class="s"> root</span>
-<span class="k">RUN</span> apt-get update <span class="se">\</span>
- <span class="o">&&</span> apt-get install -y --no-install-recommends <span class="se">\</span>
- build-essential libopenmpi-dev <span class="se">\</span>
- <span class="o">&&</span> apt-get autoremove -yqq --purge <span class="se">\</span>
- <span class="o">&&</span> apt-get clean <span class="se">\</span>
- <span class="o">&&</span> rm -rf /var/lib/apt/lists/*
-<span class="k">USER</span><span class="s"> airflow</span>
-<span class="k">RUN</span> pip install --no-cache-dir mpi4py
-</pre></div>
-</div>
-</div>
-<p>The size of this image is ~ 1.1 GB when build. As you will see further, you can achieve 20% reduction in
-size of the image in case you use "Customizing" rather than "Extending" the image.</p>
-</div>
-<div class="section" id="dag-embedding-example">
-<h3>DAG embedding example<a class="headerlink" href="#dag-embedding-example" title="Permalink to this headline">¶</a></h3>
-<p>The following example adds <code class="docutils literal notranslate"><span class="pre">test_dag.py</span></code> to your image in the <code class="docutils literal notranslate"><span class="pre">/opt/airflow/dags</span></code> folder.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/extending/embedding-dags/Dockerfile</span></p>
-<div class="highlight-Dockerfile notranslate"><div class="highlight"><pre><span></span><span class="k">FROM</span> <span class="s">apache/airflow:2.0.2</span>
-
-<span class="k">COPY</span> --chown<span class="o">=</span>airflow:root test_dag.py /opt/airflow/dags
-</pre></div>
-</div>
-</div>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/extending/embedding-dags/test_dag.py</span></p>
-<div class="highlight-Python notranslate"><div class="highlight"><pre><span></span><span class="sd">"""This dag only runs some simple tasks to test Airflow's task execution."""</span>
-<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
-
-<span class="kn">from</span> <span class="nn">airflow.models.dag</span> <span class="kn">import</span> <span class="n">DAG</span>
-<span class="kn">from</span> <span class="nn">airflow.operators.dummy</span> <span class="kn">import</span> <span class="n">DummyOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.dates</span> <span class="kn">import</span> <span class="n">days_ago</span>
-
-<span class="n">now</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span>
-<span class="n">now_to_the_hour</span> <span class="o">=</span> <span class="p">(</span><span class="n">now</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</s [...]
-<span class="n">START_DATE</span> <span class="o">=</span> <span class="n">now_to_the_hour</span>
-<span class="n">DAG_NAME</span> <span class="o">=</span> <span class="s1">'test_dag_v1'</span>
-
-<span class="n">default_args</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'owner'</span><span class="p">:</span> <span class="s1">'airflow'</span><span class="p">,</span> <span class="s1">'depends_on_past'</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span> <span class="s1">'start_date'</span><span class="p">:</span> <span class="n">days_ago</span><span class="p">(</span><span class="mi">2</span>< [...]
-<span class="n">dag</span> <span class="o">=</span> <span class="n">DAG</span><span class="p">(</span><span class="n">DAG_NAME</span><span class="p">,</span> <span class="n">schedule_interval</span><span class="o">=</span><span class="s1">'*/10 * * * *'</span><span class="p">,</span> <span class="n">default_args</span><span class="o">=</span><span class="n">default_args</span><span class="p">)</span>
-
-<span class="n">run_this_1</span> <span class="o">=</span> <span class="n">DummyOperator</span><span class="p">(</span><span class="n">task_id</span><span class="o">=</span><span class="s1">'run_this_1'</span><span class="p">,</span> <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
-<span class="n">run_this_2</span> <span class="o">=</span> <span class="n">DummyOperator</span><span class="p">(</span><span class="n">task_id</span><span class="o">=</span><span class="s1">'run_this_2'</span><span class="p">,</span> <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
-<span class="n">run_this_2</span><span class="o">.</span><span class="n">set_upstream</span><span class="p">(</span><span class="n">run_this_1</span><span class="p">)</span>
-<span class="n">run_this_3</span> <span class="o">=</span> <span class="n">DummyOperator</span><span class="p">(</span><span class="n">task_id</span><span class="o">=</span><span class="s1">'run_this_3'</span><span class="p">,</span> <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span>
-<span class="n">run_this_3</span><span class="o">.</span><span class="n">set_upstream</span><span class="p">(</span><span class="n">run_this_2</span><span class="p">)</span>
-</pre></div>
-</div>
-</div>
-</div>
-</div>
-<div class="section" id="customizing-the-image">
-<h2>Customizing the image<a class="headerlink" href="#customizing-the-image" title="Permalink to this headline">¶</a></h2>
-<p>Customizing the image is an optimized way of adding your own dependencies to the image - better
-suited to prepare highly optimized (for size) production images, especially when you have dependencies
-that require to be compiled before installing (such as <code class="docutils literal notranslate"><span class="pre">mpi4py</span></code>).</p>
-<p>It also allows more sophisticated usages, needed by "Power-users" - for example using forked version
-of Airflow, or building the images from security-vetted sources.</p>
-<p>The big advantage of this method is that it produces optimized image even if you need some compile-time
-dependencies that are not needed in the final image.</p>
-<p>The disadvantage is that you need to use Airflow Sources to build such images from the
-<a class="reference external" href="https://downloads.apache.org/airflow/">official distribution repository of Apache Airflow</a> for the
-released versions, or from the checked out sources (using release tags or main branches) in the
-<a class="reference external" href="https://github.com/apache/airflow">Airflow GitHub Project</a> or from your own fork
-if you happen to do maintain your own fork of Airflow.</p>
-<p>Another disadvantage is that the pattern of building Docker images with <code class="docutils literal notranslate"><span class="pre">--build-arg</span></code> is less familiar
-to developers of such images. However it is quite well-known to "power-users". That's why the
-customizing flow is better suited for those users who have more familiarity and have more custom
-requirements.</p>
-<p>The image also usually builds much longer than the equivalent "Extended" image because instead of
-extending the layers that are already coming from the base image, it rebuilds the layers needed
-to add extra dependencies needed at early stages of image building.</p>
-<p>When customizing the image you can choose a number of options how you install Airflow:</p>
-<blockquote>
-<div><ul class="simple">
-<li><p>From the PyPI releases (default)</p></li>
-<li><p>From the custom installation sources - using additional/replacing the original apt or PyPI repositories</p></li>
-<li><p>From local sources. This is used mostly during development.</p></li>
-<li><p>From tag or branch, or specific commit from a GitHub Airflow repository (or fork). This is particularly
-useful when you build image for a custom version of Airflow that you keep in your fork and you do not
-want to release the custom Airflow version to PyPI.</p></li>
-<li><p>From locally stored binary packages for Airflow, Airflow Providers and other dependencies. This is
-particularly useful if you want to build Airflow in a highly-secure environment where all such packages
-must be vetted by your security team and stored in your private artifact registry. This also
-allows to build airflow image in an air-gaped environment.</p></li>
-<li><p>Side note. Building <code class="docutils literal notranslate"><span class="pre">Airflow</span></code> in an <code class="docutils literal notranslate"><span class="pre">air-gaped</span></code> environment sounds pretty funny, doesn't it?</p></li>
-</ul>
-</div></blockquote>
-<p>You can also add a range of customizations while building the image:</p>
-<blockquote>
-<div><ul class="simple">
-<li><p>base python image you use for Airflow</p></li>
-<li><p>version of Airflow to install</p></li>
-<li><p>extras to install for Airflow (or even removing some default extras)</p></li>
-<li><p>additional apt/python dependencies to use while building Airflow (DEV dependencies)</p></li>
-<li><p>additional apt/python dependencies to install for runtime version of Airflow (RUNTIME dependencies)</p></li>
-<li><p>additional commands and variables to set if needed during building or preparing Airflow runtime</p></li>
-<li><p>choosing constraint file to use when installing Airflow</p></li>
-</ul>
-</div></blockquote>
-<p>Additional explanation is needed for the last point. Airflow uses constraints to make sure
-that it can be predictably installed, even if some new versions of Airflow dependencies are
-released (or even dependencies of our dependencies!). The docker image and accompanying scripts
-usually determine automatically the right versions of constraints to be used based on the Airflow
-version installed and Python version. For example 2.0.2 version of Airflow installed from PyPI
-uses constraints from <code class="docutils literal notranslate"><span class="pre">constraints-2.0.2</span></code> tag). However in some cases - when installing airflow from
-GitHub for example - you have to manually specify the version of constraints used, otherwise
-it will default to the latest version of the constraints which might not be compatible with the
-version of Airflow you use.</p>
-<p>You can also download any version of Airflow constraints and adapt it with your own set of
-constraints and manually set your own versions of dependencies in your own constraints and use the version
-of constraints that you manually prepared.</p>
-<p>You can read more about constraints in the documentation of the
-<a class="reference external" href="http://airflow.apache.org/docs/apache-airflow/stable/installation.html#constraints-files">Installation</a></p>
-</div>
-<div class="section" id="examples-of-image-customizing">
-<h2>Examples of image customizing<a class="headerlink" href="#examples-of-image-customizing" title="Permalink to this headline">¶</a></h2>
-<div class="section" id="building-from-pypi-packages">
-<span id="image-build-pypi"></span><h3>Building from PyPI packages<a class="headerlink" href="#building-from-pypi-packages" title="Permalink to this headline">¶</a></h3>
-<p>This is the basic way of building the custom images from sources.</p>
-<p>The following example builds the production image in version <code class="docutils literal notranslate"><span class="pre">3.6</span></code> with latest PyPI-released Airflow,
-with default set of Airflow extras and dependencies. The <code class="docutils literal notranslate"><span class="pre">2.0.2</span></code> constraints are used automatically.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/stable-airflow.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-<p>The following example builds the production image in version <code class="docutils literal notranslate"><span class="pre">3.7</span></code> with default extras from <code class="docutils literal notranslate"><span class="pre">2.0.2</span></code> PyPI
-package. The <code class="docutils literal notranslate"><span class="pre">2.0.2</span></code> constraints are used automatically.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/pypi-selected-version.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.7-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_VERSION</span><span class="o">=</span><span class="s2">"2.0.2"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-<p>The following example builds the production image in version <code class="docutils literal notranslate"><span class="pre">3.8</span></code> with additional airflow extras
-(<code class="docutils literal notranslate"><span class="pre">mssql,hdfs</span></code>) from <code class="docutils literal notranslate"><span class="pre">2.0.2</span></code> PyPI package, and additional dependency (<code class="docutils literal notranslate"><span class="pre">oauth2client</span></code>).</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/pypi-extras-and-deps.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.8-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_VERSION</span><span class="o">=</span><span class="s2">"2.0.2"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_AIRFLOW_EXTRAS</span><span class="o">=</span><span class="s2">"mssql,hdfs"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_PYTHON_DEPS</span><span class="o">=</span><span class="s2">"oauth2client"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-<p>The following example adds <code class="docutils literal notranslate"><span class="pre">mpi4py</span></code> package which requires both <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> and <code class="docutils literal notranslate"><span class="pre">mpi</span> <span class="pre">compiler</span></code>.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/add-build-essential-custom.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.6-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_VERSION</span><span class="o">=</span><span class="s2">"2.0.2"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_PYTHON_DEPS</span><span class="o">=</span><span class="s2">"mpi4py"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_DEV_APT_DEPS</span><span class="o">=</span><span class="s2">"libopenmpi-dev"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_RUNTIME_APT_DEPS</span><span class="o">=</span><span class="s2">"openmpi-common"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-<p>The above image is equivalent of the "extended" image from previous chapter but it's size is only
-874 MB. Comparing to 1.1 GB of the "extended image" this is about 230 MB less, so you can achieve ~20%
-improvement in size of the image by using "customization" vs. extension. The saving can increase in case you
-have more complex dependencies to build.</p>
-</div>
-<div class="section" id="building-optimized-images">
-<span id="image-build-optimized"></span><h3>Building optimized images<a class="headerlink" href="#building-optimized-images" title="Permalink to this headline">¶</a></h3>
-<p>The following example the production image in version <code class="docutils literal notranslate"><span class="pre">3.6</span></code> with additional airflow extras from <code class="docutils literal notranslate"><span class="pre">2.0.2</span></code>
-PyPI package but it includes additional apt dev and runtime dependencies.</p>
-<p>The dev dependencies are those that require <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> and usually need to involve recompiling
-of some python dependencies so those packages might require some additional DEV dependencies to be
-present during recompilation. Those packages are not needed at runtime, so we only install them for the
-"build" time. They are not installed in the final image, thus producing much smaller images.
-In this case pandas requires recompilation so it also needs gcc and g++ as dev APT dependencies.
-The <code class="docutils literal notranslate"><span class="pre">jre-headless</span></code> does not require recompiling so it can be installed as the runtime APT dependency.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/pypi-dev-runtime-deps.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.6-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_VERSION</span><span class="o">=</span><span class="s2">"2.0.2"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_AIRFLOW_EXTRAS</span><span class="o">=</span><span class="s2">"jdbc"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_PYTHON_DEPS</span><span class="o">=</span><span class="s2">"pandas"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_DEV_APT_DEPS</span><span class="o">=</span><span class="s2">"gcc g++"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_RUNTIME_APT_DEPS</span><span class="o">=</span><span class="s2">"default-jre-headless"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-</div>
-<div class="section" id="building-from-github">
-<span id="image-build-github"></span><h3>Building from GitHub<a class="headerlink" href="#building-from-github" title="Permalink to this headline">¶</a></h3>
-<p>This method is usually used for development purpose. But in case you have your own fork you can point
-it to your forked version of source code without having to release it to PyPI. It is enough to have
-a branch or tag in your repository and use the tag or branch in the URL that you point the installation to.</p>
-<p>In case of GitHyb builds you need to pass the constraints reference manually in case you want to use
-specific constraints, otherwise the default <code class="docutils literal notranslate"><span class="pre">constraints-master</span></code> is used.</p>
-<p>The following example builds the production image in version <code class="docutils literal notranslate"><span class="pre">3.7</span></code> with default extras from the latest master version and
-constraints are taken from latest version of the constraints-master branch in GitHub.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/github-master.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.7-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_INSTALLATION_METHOD</span><span class="o">=</span><span class="s2">"https://github.com/apache/airflow/archive/master.tar.gz#egg=apache-airflow"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_CONSTRAINTS_REFERENCE</span><span class="o">=</span><span class="s2">"constraints-master"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-<p>The following example builds the production image with default extras from the
-latest <code class="docutils literal notranslate"><span class="pre">v2-0-test</span></code> version and constraints are taken from the latest version of
-the <code class="docutils literal notranslate"><span class="pre">constraints-2-0</span></code> branch in GitHub. Note that this command might fail occasionally as only
-the "released version" constraints when building a version and "master" constraints when building
-master are guaranteed to work.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/github-v2-0-test.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.8-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_INSTALLATION_METHOD</span><span class="o">=</span><span class="s2">"https://github.com/apache/airflow/archive/v2-0-test.tar.gz#egg=apache-airflow"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_CONSTRAINTS_REFERENCE</span><span class="o">=</span><span class="s2">"constraints-2-0"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-<p>You can also specify another repository to build from. If you also want to use different constraints
-repository source, you must specify it as additional <code class="docutils literal notranslate"><span class="pre">CONSTRAINTS_GITHUB_REPOSITORY</span></code> build arg.</p>
-<p>The following example builds the production image using <code class="docutils literal notranslate"><span class="pre">potiuk/airflow</span></code> fork of Airflow and constraints
-are also downloaded from that repository.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/github-different-repository.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.8-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_INSTALLATION_METHOD</span><span class="o">=</span><span class="s2">"https://github.com/potiuk/airflow/archive/master.tar.gz#egg=apache-airflow"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_CONSTRAINTS_REFERENCE</span><span class="o">=</span><span class="s2">"constraints-master"</span> <span class="se">\</span>
- --build-arg <span class="nv">CONSTRAINTS_GITHUB_REPOSITORY</span><span class="o">=</span><span class="s2">"potiuk/airflow"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-</div>
-<div class="section" id="using-custom-installation-sources">
-<span id="image-build-custom"></span><h3>Using custom installation sources<a class="headerlink" href="#using-custom-installation-sources" title="Permalink to this headline">¶</a></h3>
-<p>You can customize more aspects of the image - such as additional commands executed before apt dependencies
-are installed, or adding extra sources to install your dependencies from. You can see all the arguments
-described below but here is an example of rather complex command to customize the image
-based on example in <a class="reference external" href="https://github.com/apache/airflow/issues/8605#issuecomment-690065621">this comment</a>:</p>
-<p>In case you need to use your custom PyPI package indexes, you can also customize PYPI sources used during
-image build by adding a <code class="docutils literal notranslate"><span class="pre">docker-context-files</span></code>/<code class="docutils literal notranslate"><span class="pre">.pypirc</span></code> file when building the image.
-This <code class="docutils literal notranslate"><span class="pre">.pypirc</span></code> will not be committed to the repository (it is added to <code class="docutils literal notranslate"><span class="pre">.gitignore</span></code>) and it will not be
-present in the final production image. It is added and used only in the build segment of the image.
-Therefore this <code class="docutils literal notranslate"><span class="pre">.pypirc</span></code> file can safely contain list of package indexes you want to use,
-usernames and passwords used for authentication. More details about <code class="docutils literal notranslate"><span class="pre">.pypirc</span></code> file can be found in the
-<a class="reference external" href="https://packaging.python.org/specifications/pypirc/">pypirc specification</a>.</p>
-<p>Such customizations are independent of the way how airflow is installed.</p>
-<div class="admonition note">
-<p class="admonition-title">Note</p>
-<p>Similar results could be achieved by modifying the Dockerfile manually (see below) and injecting the
-commands needed, but by specifying the customizations via build-args, you avoid the need of
-synchronizing the changes from future Airflow Dockerfiles. Those customizations should work with the
-future version of Airflow's official <code class="docutils literal notranslate"><span class="pre">Dockerfile</span></code> at most with minimal modifications od parameter
-names (if any), so using the build command for your customizations makes your custom image more
-future-proof.</p>
-</div>
-<p>The following - rather complex - example shows capabilities of:</p>
-<blockquote>
-<div><ul class="simple">
-<li><p>Adding airflow extras (slack, odbc)</p></li>
-<li><p>Adding PyPI dependencies (<code class="docutils literal notranslate"><span class="pre">azure-storage-blob,</span> <span class="pre">oauth2client,</span> <span class="pre">beautifulsoup4,</span> <span class="pre">dateparser,</span> <span class="pre">rocketchat_API,typeform</span></code>)</p></li>
-<li><p>Adding custom environment variables while installing <code class="docutils literal notranslate"><span class="pre">apt</span></code> dependencies - both DEV and RUNTIME
-(<code class="docutils literal notranslate"><span class="pre">ACCEPT_EULA=Y'</span></code>)</p></li>
-<li><p>Adding custom curl command for adding keys and configuring additional apt sources needed to install
-<code class="docutils literal notranslate"><span class="pre">apt</span></code> dependencies (both DEV and RUNTIME)</p></li>
-<li><p>Adding custom <code class="docutils literal notranslate"><span class="pre">apt</span></code> dependencies, both DEV (<code class="docutils literal notranslate"><span class="pre">msodbcsql17</span> <span class="pre">unixodbc-dev</span> <span class="pre">g++)</span> <span class="pre">and</span> <span class="pre">runtime</span> <span class="pre">msodbcsql17</span> <span class="pre">unixodbc</span> <span class="pre">git</span> <span class="pre">procps</span> <span class="pre">vim</spa [...]
-</ul>
-</div></blockquote>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/customizing/custom-sources.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . -f Dockerfile <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.7-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_VERSION</span><span class="o">=</span><span class="s2">"2.0.2"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_AIRFLOW_EXTRAS</span><span class="o">=</span><span class="s2">"slack,odbc"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_PYTHON_DEPS</span><span class="o">=</span><span class="s2">" \</span>
-<span class="s2"> azure-storage-blob \</span>
-<span class="s2"> oauth2client \</span>
-<span class="s2"> beautifulsoup4 \</span>
-<span class="s2"> dateparser \</span>
-<span class="s2"> rocketchat_API \</span>
-<span class="s2"> typeform"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_DEV_APT_COMMAND</span><span class="o">=</span><span class="s2">"curl https://packages.microsoft.com/keys/microsoft.asc | \</span>
-<span class="s2"> apt-key add --no-tty - && \</span>
-<span class="s2"> curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_DEV_APT_ENV</span><span class="o">=</span><span class="s2">"ACCEPT_EULA=Y"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_DEV_APT_DEPS</span><span class="o">=</span><span class="s2">"msodbcsql17 unixodbc-dev g++"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_RUNTIME_APT_COMMAND</span><span class="o">=</span><span class="s2">"curl https://packages.microsoft.com/keys/microsoft.asc | \</span>
-<span class="s2"> apt-key add --no-tty - && \</span>
-<span class="s2"> curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_RUNTIME_APT_ENV</span><span class="o">=</span><span class="s2">"ACCEPT_EULA=Y"</span> <span class="se">\</span>
- --build-arg <span class="nv">ADDITIONAL_RUNTIME_APT_DEPS</span><span class="o">=</span><span class="s2">"msodbcsql17 unixodbc git procps vim"</span> <span class="se">\</span>
- --tag <span class="s2">"</span><span class="k">$(</span>basename <span class="s2">"</span><span class="nv">$0</span><span class="s2">"</span><span class="k">)</span><span class="s2">"</span>
-</pre></div>
-</div>
-</div>
-</div>
-<div class="section" id="build-images-in-security-restricted-environments">
-<span id="image-build-secure-environments"></span><h3>Build images in security restricted environments<a class="headerlink" href="#build-images-in-security-restricted-environments" title="Permalink to this headline">¶</a></h3>
-<p>You can also make sure your image is only build using local constraint file and locally downloaded
-wheel files. This is often useful in Enterprise environments where the binary files are verified and
-vetted by the security teams. It is also the most complex way of building the image. You should be an
-expert of building and using Dockerfiles in order to use it and have to have specific needs of security if
-you want to follow that route.</p>
-<p>This builds below builds the production image with packages and constraints used from the local
-<code class="docutils literal notranslate"><span class="pre">docker-context-files</span></code> rather than installed from PyPI or GitHub. It also disables MySQL client
-installation as it is using external installation method.</p>
-<p>Note that as a prerequisite - you need to have downloaded wheel files. In the example below we
-first download such constraint file locally and then use <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">download</span></code> to get the <code class="docutils literal notranslate"><span class="pre">.whl</span></code> files needed
-but in most likely scenario, those wheel files should be copied from an internal repository of such .whl
-files. Note that <code class="docutils literal notranslate"><span class="pre">AIRFLOW_VERSION_SPECIFICATION</span></code> is only there for reference, the apache airflow <code class="docutils literal notranslate"><span class="pre">.whl</span></code> file
-in the right version is part of the <code class="docutils literal notranslate"><span class="pre">.whl</span></code> files downloaded.</p>
-<p>Note that 'pip download' will only works on Linux host as some of the packages need to be compiled from
-sources and you cannot install them providing <code class="docutils literal notranslate"><span class="pre">--platform</span></code> switch. They also need to be downloaded using
-the same python version as the target image.</p>
-<p>The <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">download</span></code> might happen in a separate environment. The files can be committed to a separate
-binary repository and vetted/verified by the security team and used subsequently to build images
-of Airflow when needed on an air-gaped system.</p>
-<p>Example of preparing the constraint files and wheel files. Note that <code class="docutils literal notranslate"><span class="pre">mysql</span></code> dependency is removed
-as <code class="docutils literal notranslate"><span class="pre">mysqlclient</span></code> is installed from Oracle's <code class="docutils literal notranslate"><span class="pre">apt</span></code> repository and if you want to add it, you need
-to provide this library from you repository if you want to build Airflow image in an "air-gaped" system.</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/restricted/restricted_environments.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>rm docker-context-files/*.whl docker-context-files/*.tar.gz docker-context-files/*.txt <span class="o">||</span> <span class="nb">true</span>
-
-curl -Lo <span class="s2">"docker-context-files/constraints-3.7.txt"</span> <span class="se">\</span>
- https://raw.githubusercontent.com/apache/airflow/constraints-2.0.2/constraints-3.7.txt
-
-pip download --dest docker-context-files <span class="se">\</span>
- --constraint docker-context-files/constraints-3.7.txt <span class="se">\</span>
- <span class="s2">"apache-airflow[async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,postgres,redis,slack,ssh,statsd,virtualenv]==2.0.2"</span>
-</pre></div>
-</div>
-</div>
-<p>After this step is finished, your <code class="docutils literal notranslate"><span class="pre">docker-context-files</span></code> folder will contain all the packages that
-are needed to install Airflow from.</p>
-<p>Those downloaded packages and constraint file can be pre-vetted by your security team before you attempt
-to install the image. You can also store those downloaded binary packages in your private artifact registry
-which allows for the flow where you will download the packages on one machine, submit only new packages for
-security vetting and only use the new packages when they were vetted.</p>
-<p>On a separate (air-gaped) system, all the PyPI packages can be copied to <code class="docutils literal notranslate"><span class="pre">docker-context-files</span></code>
-where you can build the image using the packages downloaded by passing those build args:</p>
-<blockquote>
-<div><ul class="simple">
-<li><p><code class="docutils literal notranslate"><span class="pre">INSTALL_FROM_DOCKER_CONTEXT_FILES="true"</span></code> - to use packages present in <code class="docutils literal notranslate"><span class="pre">docker-context-files</span></code></p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW_PRE_CACHED_PIP_PACKAGES="false"</span></code> - to not pre-cache packages from PyPI when building image</p></li>
-<li><p><code class="docutils literal notranslate"><span class="pre">AIRFLOW_CONSTRAINTS_LOCATION=/docker-context-files/YOUR_CONSTRAINT_FILE.txt</span></code> - to downloaded constraint files</p></li>
-<li><p>(Optional) <code class="docutils literal notranslate"><span class="pre">INSTALL_MYSQL_CLIENT="false"</span></code> if you do not want to install <code class="docutils literal notranslate"><span class="pre">MySQL</span></code>
-client from the Oracle repositories. In this case also make sure that your</p></li>
-</ul>
-</div></blockquote>
-<p>Note, that the solution we have for installing python packages from local packages, only solves the problem
-of "air-gaped" python installation. The Docker image also downloads <code class="docutils literal notranslate"><span class="pre">apt</span></code> dependencies and <code class="docutils literal notranslate"><span class="pre">node-modules</span></code>.
-Those type of dependencies are however more likely to be available in your "air-gaped" system via transparent
-proxies and it should automatically reach out to your private registries, however in the future the
-solution might be applied to both of those installation steps.</p>
-<p>You can also use techniques described in the previous chapter to make <code class="docutils literal notranslate"><span class="pre">docker</span> <span class="pre">build</span></code> use your private
-apt sources or private PyPI repositories (via <code class="docutils literal notranslate"><span class="pre">.pypirc</span></code>) available which can be security-vetted.</p>
-<p>If you fulfill all the criteria, you can build the image on an air-gaped system by running command similar
-to the below:</p>
-<div class="example-block-wrapper docutils container">
-<p class="example-header"><span class="example-title">docs/docker-stack/docker-examples/restricted/restricted_environments.sh</span></p>
-<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>docker build . <span class="se">\</span>
- --build-arg <span class="nv">PYTHON_BASE_IMAGE</span><span class="o">=</span><span class="s2">"python:3.7-slim-buster"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_INSTALLATION_METHOD</span><span class="o">=</span><span class="s2">"apache-airflow"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_VERSION</span><span class="o">=</span><span class="s2">"2.0.2"</span> <span class="se">\</span>
- --build-arg <span class="nv">INSTALL_MYSQL_CLIENT</span><span class="o">=</span><span class="s2">"false"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_PRE_CACHED_PIP_PACKAGES</span><span class="o">=</span><span class="s2">"false"</span> <span class="se">\</span>
- --build-arg <span class="nv">INSTALL_FROM_DOCKER_CONTEXT_FILES</span><span class="o">=</span><span class="s2">"true"</span> <span class="se">\</span>
- --build-arg <span class="nv">AIRFLOW_CONSTRAINTS_LOCATION</span><span class="o">=</span><span class="s2">"/docker-context-files/constraints-3.7.txt"</span>
-</pre></div>
-</div>
-</div>
-</div>
-<div class="section" id="modifying-the-dockerfile">
-<h3>Modifying the Dockerfile<a class="headerlink" href="#modifying-the-dockerfile" title="Permalink to this headline">¶</a></h3>
-<p>The build arg approach is a convenience method if you do not want to manually modify the <code class="docutils literal notranslate"><span class="pre">Dockerfile</span></code>.
-Our approach is flexible enough, to be able to accommodate most requirements and
-customizations out-of-the-box. When you use it, you do not need to worry about adapting the image every
-time new version of Airflow is released. However sometimes it is not enough if you have very
-specific needs and want to build a very custom image. In such case you can simply modify the
-<code class="docutils literal notranslate"><span class="pre">Dockerfile</span></code> manually as you see fit and store it in your forked repository. However you will have to
-make sure to rebase your changes whenever new version of Airflow is released, because we might modify
-the approach of our Dockerfile builds in the future and you might need to resolve conflicts
-and rebase your changes.</p>
-<p>There are a few things to remember when you modify the <code class="docutils literal notranslate"><span class="pre">Dockerfile</span></code>:</p>
-<ul>
-<li><p>We are using the widely recommended pattern of <code class="docutils literal notranslate"><span class="pre">.dockerignore</span></code> where everything is ignored by default
-and only the required folders are added through exclusion (!). This allows to keep docker context small
-because there are many binary artifacts generated in the sources of Airflow and if they are added to
-the context, the time of building the image would increase significantly. If you want to add any new
-folders to be available in the image you must add it here with leading <code class="docutils literal notranslate"><span class="pre">!</span></code>.</p>
-<div class="highlight-text notranslate"><div class="highlight"><pre><span></span># Ignore everything
-**
-
-# Allow only these directories
-!airflow
-...
-</pre></div>
-</div>
-</li>
-<li><p>The <code class="docutils literal notranslate"><span class="pre">docker-context-files</span></code> folder is automatically added to the context of the image, so if you want
-to add individual files, binaries, requirement files etc you can add them there. The
-<code class="docutils literal notranslate"><span class="pre">docker-context-files</span></code> is copied to the <code class="docutils literal notranslate"><span class="pre">/docker-context-files</span></code> folder of the build segment of the
-image, so it is not present in the final image - which makes the final image smaller in case you want
-to use those files only in the <code class="docutils literal notranslate"><span class="pre">build</span></code> segment. You must copy any files from the directory manually,
-using COPY command if you want to get the files in your final image (in the main image segment).</p></li>
+ <div class="section" id="module-airflow.executors.kubernetes_executor">
+<span id="airflow-executors-kubernetes-executor"></span><h1><a class="reference internal" href="#module-airflow.executors.kubernetes_executor" title="airflow.executors.kubernetes_executor"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a><a class="headerlink" href="#module-airflow.executors.kubernetes_executor" title="Permalink to this headline">¶</a></h1>
+<p><p>KubernetesExecutor</p>
+<div class="admonition seealso">
+<p class="admonition-title">See also</p>
+<p>For more information on how the KubernetesExecutor works, take a look at the guide:
+<a class="reference internal" href="../../../../executor/kubernetes.html#executor-kubernetesexecutor"><span class="std std-ref">Kubernetes Executor</span></a></p>
+</div>
+</p>
+<div class="section" id="module-contents">
+<h2>Module Contents<a class="headerlink" href="#module-contents" title="Permalink to this headline">¶</a></h2>
+<dl class="py data">
+<dt id="airflow.executors.kubernetes_executor.KubernetesJobType">
+<code class="sig-prename descclassname">airflow.executors.kubernetes_executor.</code><code class="sig-name descname">KubernetesJobType</code><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesJobType"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.KubernetesJobType" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.kubernetes_executor.KubernetesResultsType">
+<code class="sig-prename descclassname">airflow.executors.kubernetes_executor.</code><code class="sig-name descname">KubernetesResultsType</code><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesResultsType"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.KubernetesResultsType" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py data">
+<dt id="airflow.executors.kubernetes_executor.KubernetesWatchType">
+<code class="sig-prename descclassname">airflow.executors.kubernetes_executor.</code><code class="sig-name descname">KubernetesWatchType</code><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesWatchType"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.KubernetesWatchType" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py class">
+<dt id="airflow.executors.kubernetes_executor.ResourceVersion">
+<em class="property">class </em><code class="sig-prename descclassname">airflow.executors.kubernetes_executor.</code><code class="sig-name descname">ResourceVersion</code><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#ResourceVersion"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.ResourceVersion" title="Permalink to this definition">¶</a></dt>
+<dd><p>Singleton for tracking resourceVersion from Kubernetes</p>
+<dl class="py attribute">
+<dt id="airflow.executors.kubernetes_executor.ResourceVersion._instance">
+<code class="sig-name descname">_instance</code><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#ResourceVersion._instance"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.ResourceVersion._instance" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py attribute">
+<dt id="airflow.executors.kubernetes_executor.ResourceVersion.resource_version">
+<code class="sig-name descname">resource_version</code><em class="property"> = 0</em><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#ResourceVersion.resource_version"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.ResourceVersion.resource_version" title="Permalink to this definition">¶</a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.ResourceVersion.__new__">
+<em class="property">classmethod </em><code class="sig-name descname">__new__</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">cls</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#ResourceVersion.__new__"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.ResourceVersion.__new__" title="Permalink to this de [...]
+<dd></dd></dl>
+
+</dd></dl>
+
+<dl class="py class">
+<dt id="airflow.executors.kubernetes_executor.KubernetesJobWatcher">
+<em class="property">class </em><code class="sig-prename descclassname">airflow.executors.kubernetes_executor.</code><code class="sig-name descname">KubernetesJobWatcher</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">namespace</span><span class="p">:</span> <span class="n">Optional<span class="p">[</span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a><span class="p">]</span></span></em> [...]
+<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Process" title="(in Python v3.9)"><code class="xref py py-class docutils literal notranslate"><span class="pre">multiprocessing.Process</span></code></a>, <code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.utils.log.logging_mixin.LoggingMixin</span></code></p>
+<p>Watches for Kubernetes jobs</p>
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesJobWatcher.run">
+<code class="sig-name descname">run</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesJobWatcher.run"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.KubernetesJobWatcher.run" title="Permalink to this definition">¶</a></dt>
+<dd><p>Performs watching</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesJobWatcher._run">
+<code class="sig-name descname">_run</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">kube_client</span><span class="p">:</span> <span class="n">client.CoreV1Api</span></em>, <em class="sig-param"><span class="n">resource_version</span><span class="p">:</span> <span class="n">Optional<span class="p">[</span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesJobWatcher.process_error">
+<code class="sig-name descname">process_error</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">event</span><span class="p">:</span> <span class="n">Any</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesJobWatcher.process_error"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#ai [...]
+<dd><p>Process error response</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesJobWatcher.process_status">
+<code class="sig-name descname">process_status</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">pod_id</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a></span></em>, <em class="sig-param"><span class="n">namespace</span><span class="p">:</span> <span class="n"><a class="reference external" [...]
+<dd><p>Process status response</p>
+</dd></dl>
+
+</dd></dl>
+
+<dl class="py class">
+<dt id="airflow.executors.kubernetes_executor.AirflowKubernetesScheduler">
+<em class="property">class </em><code class="sig-prename descclassname">airflow.executors.kubernetes_executor.</code><code class="sig-name descname">AirflowKubernetesScheduler</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">kube_config</span><span class="p">:</span> <span class="n">Any</span></em>, <em class="sig-param"><span class="n">task_queue</span><span class="p">:</span> <span class="n">'Queue[KubernetesJobType]'</span></em>, <em class="sig-param"><span [...]
+<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.utils.log.logging_mixin.LoggingMixin</span></code></p>
+<p>Airflow Scheduler for Kubernetes</p>
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.run_pod_async">
+<code class="sig-name descname">run_pod_async</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">pod</span><span class="p">:</span> <span class="n">k8s.V1Pod</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#AirflowKubernetesScheduler. [...]
+<dd><p>Runs POD asynchronously</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.AirflowKubernetesScheduler._make_kube_watcher">
+<code class="sig-name descname">_make_kube_watcher</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#AirflowKubernetesScheduler._make_kube_watcher"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler._make_kube_watcher" title="Pe [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.AirflowKubernetesScheduler._health_check_kube_watcher">
+<code class="sig-name descname">_health_check_kube_watcher</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#AirflowKubernetesScheduler._health_check_kube_watcher"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler._health_check [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.run_next">
+<code class="sig-name descname">run_next</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">next_job</span><span class="p">:</span> <span class="n"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobType" title="airflow.executors.kubernetes_executor.KubernetesJobType">KubernetesJobType</a></span></em><span class="sig-paren">)</span><a class="reference internal" href="../. [...]
+<dd><p>The run_next command will check the task_queue for any un-run jobs.
+It will then create a unique job-id, launch that job in the cluster,
+and store relevant info in the current_jobs map so we can track the job's
+status</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.delete_pod">
+<code class="sig-name descname">delete_pod</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">pod_id</span><span class="p">:</span> <span class="n"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a></span></em>, <em class="sig-param"><span class="n">namespace</span><span class="p">:</span> <span class="n"><a class="reference external" hre [...]
+<dd><p>Deletes POD</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.sync">
+<code class="sig-name descname">sync</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#AirflowKubernetesScheduler.sync"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.sync" title="Permalink to this definition">¶</a></dt>
+<dd><p>The sync function checks the status of all currently running kubernetes jobs.
+If a job is completed, its status is placed in the result queue to
+be sent back to the scheduler.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p></p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.process_watcher_task">
+<code class="sig-name descname">process_watcher_task</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">task</span><span class="p">:</span> <span class="n"><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesWatchType" title="airflow.executors.kubernetes_executor.KubernetesWatchType">KubernetesWatchType</a></span></em><span class="sig-paren">)</span><a class="reference intern [...]
+<dd><p>Process the task by watcher.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.AirflowKubernetesScheduler._flush_watcher_queue">
+<code class="sig-name descname">_flush_watcher_queue</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#AirflowKubernetesScheduler._flush_watcher_queue"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler._flush_watcher_queue" tit [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.terminate">
+<code class="sig-name descname">terminate</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#AirflowKubernetesScheduler.terminate"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.AirflowKubernetesScheduler.terminate" title="Permalink to this definition" [...]
+<dd><p>Terminates the watcher.</p>
+</dd></dl>
+
+</dd></dl>
+
+<dl class="py function">
+<dt id="airflow.executors.kubernetes_executor.get_base_pod_from_template">
+<code class="sig-prename descclassname">airflow.executors.kubernetes_executor.</code><code class="sig-name descname">get_base_pod_from_template</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">pod_template_file</span><span class="p">:</span> <span class="n">Optional<span class="p">[</span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9)">str</a><span class="p">]</span></span></em>, <em class="sig-p [...]
+<dt>
+<code class="sig-name descname">Reads either the pod_template_file set in the executor_config or the base pod_template_file</code></dt>
+<dt>
+<code class="sig-name descname">set in the airflow.cfg to craft a "base pod" that will be used by the KubernetesExecutor</code></dt>
+<dd><dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>pod_template_file</strong> -- absolute path to a pod_template_file.yaml or None</p></li>
+<li><p><strong>kube_config</strong> -- The KubeConfig class generated by airflow that contains all kube metadata</p></li>
</ul>
-</div>
-</div>
-<div class="section" id="more-details">
-<h2>More details<a class="headerlink" href="#more-details" title="Permalink to this headline">¶</a></h2>
-<div class="section" id="build-args-reference">
-<h3>Build Args reference<a class="headerlink" href="#build-args-reference" title="Permalink to this headline">¶</a></h3>
-<p>The detailed <code class="docutils literal notranslate"><span class="pre">--build-arg</span></code> reference can be found in <a class="reference internal" href="build-arg-ref.html"><span class="doc">Image build arguments reference</span></a>.</p>
-</div>
-<div class="section" id="the-architecture-of-the-images">
-<h3>The architecture of the images<a class="headerlink" href="#the-architecture-of-the-images" title="Permalink to this headline">¶</a></h3>
-<p>You can read more details about the images - the context, their parameters and internal structure in the
-<a class="reference external" href="https://github.com/apache/airflow/blob/master/IMAGES.rst">IMAGES.rst</a> document.</p>
-</div>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>a V1Pod that can be used as the base pod for k8s tasks</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py class">
+<dt id="airflow.executors.kubernetes_executor.KubernetesExecutor">
+<em class="property">class </em><code class="sig-prename descclassname">airflow.executors.kubernetes_executor.</code><code class="sig-name descname">KubernetesExecutor</code><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesExecutor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.KubernetesExecutor" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="../base_executor/index.html#airflow.executors.base_executor.BaseExecutor" title="airflow.executors.base_executor.BaseExecutor"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.executors.base_executor.BaseExecutor</span></code></a>, <code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.utils.log.logging_mixin.LoggingMixin</span></code></p>
+<p>Executor for Kubernetes</p>
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesExecutor.clear_not_launched_queued_tasks">
+<code class="sig-name descname">clear_not_launched_queued_tasks</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">session</span><span class="o">=</span><span class="default_value">None</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesExecutor.clear_not_launched_queued_tasks"><span class="viewcode-link">[s [...]
+<dd><p>If the airflow scheduler restarts with pending "Queued" tasks, the tasks may or
+may not
+have been launched. Thus on starting up the scheduler let's check every
+"Queued" task to
+see if it has been launched (ie: if there is a corresponding pod on kubernetes)</p>
+<p>If it has been launched then do nothing, otherwise reset the state to "None" so
+the task
+will be rescheduled</p>
+<p>This will not be necessary in a future version of airflow in which there is
+proper support
+for State.LAUNCHED</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesExecutor.start">
+<code class="sig-name descname">start</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesExecutor.start"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.start" title="Permalink to this definition">¶</a></dt>
+<dd><p>Starts the executor</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesExecutor.execute_async">
+<code class="sig-name descname">execute_async</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">key</span><span class="p">:</span> <span class="n">TaskInstanceKey</span></em>, <em class="sig-param"><span class="n">command</span><span class="p">:</span> <span class="n">CommandType</span></em>, <em class="sig-param"><span class="n">queue</span><span class="p">:</span> <span class="n">Optional<span class="p">[ [...]
+<dd><p>Executes task asynchronously</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesExecutor.sync">
+<code class="sig-name descname">sync</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesExecutor.sync"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.sync" title="Permalink to this definition">¶</a></dt>
+<dd><p>Synchronize task state.</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesExecutor._check_worker_pods_pending_timeout">
+<code class="sig-name descname">_check_worker_pods_pending_timeout</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesExecutor._check_worker_pods_pending_timeout"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.KubernetesExecutor._check_worker [...]
+<dd><p>Check if any pending worker pods have timed out</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesExecutor._change_state">
+<code class="sig-name descname">_change_state</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">key</span><span class="p">:</span> <span class="n">TaskInstanceKey</span></em>, <em class="sig-param"><span class="n">state</span><span class="p">:</span> <span class="n">Optional<span class="p">[</span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.9 [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesExecutor.try_adopt_task_instances">
+<code class="sig-name descname">try_adopt_task_instances</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">tis</span><span class="p">:</span> <span class="n">List<span class="p">[</span>TaskInstance<span class="p">]</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesExecutor.try_adopt_task_instances" [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesExecutor.adopt_launched_task">
+<code class="sig-name descname">adopt_launched_task</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">kube_client</span><span class="p">:</span> <span class="n">client.CoreV1Api</span></em>, <em class="sig-param"><span class="n">pod</span><span class="p">:</span> <span class="n">k8s.V1Pod</span></em>, <em class="sig-param"><span class="n">pod_ids</span><span class="p">:</span> <span class="n">Dict<span clas [...]
+<dd><p>Patch existing pod so that the current KubernetesJobWatcher can monitor it via label selectors</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>kube_client</strong> -- kubernetes client for speaking to kube API</p></li>
+<li><p><strong>pod</strong> -- V1Pod spec that we will patch with new label</p></li>
+<li><p><strong>pod_ids</strong> -- pod_ids we expect to patch.</p></li>
+</ul>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesExecutor._adopt_completed_pods">
+<code class="sig-name descname">_adopt_completed_pods</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em>, <em class="sig-param"><span class="n">kube_client</span><span class="p">:</span> <span class="n">client.CoreV1Api</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesExecutor._adopt_completed_pods"><span class="viewcode-link">[source]</span></ [...]
+<dd><p>Patch completed pod so that the KubernetesJobWatcher can delete it.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>kube_client</strong> -- kubernetes client for speaking to kube API</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesExecutor._flush_task_queue">
+<code class="sig-name descname">_flush_task_queue</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesExecutor._flush_task_queue"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.KubernetesExecutor._flush_task_queue" title="Permalink to this def [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesExecutor._flush_result_queue">
+<code class="sig-name descname">_flush_result_queue</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesExecutor._flush_result_queue"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.KubernetesExecutor._flush_result_queue" title="Permalink to th [...]
+<dd></dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesExecutor.end">
+<code class="sig-name descname">end</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesExecutor.end"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.end" title="Permalink to this definition">¶</a></dt>
+<dd><p>Called when the executor shuts down</p>
+</dd></dl>
+
+<dl class="py method">
+<dt id="airflow.executors.kubernetes_executor.KubernetesExecutor.terminate">
+<code class="sig-name descname">terminate</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">self</span></em><span class="sig-paren">)</span><a class="reference internal" href="../../../../_modules/airflow/executors/kubernetes_executor.html#KubernetesExecutor.terminate"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.executors.kubernetes_executor.KubernetesExecutor.terminate" title="Permalink to this definition">¶</a></dt>
+<dd><p>Terminate the executor is not doing anything.</p>
+</dd></dl>
+
+</dd></dl>
+
</div>
</div>
<div class="pager" role="navigation" aria-label="related navigation">
- <a rel="prev" title="Docker Image for Apache Airflow" href="index.html" >
+ <a rel="prev" title="airflow.executors.executor_loader" href="../executor_loader/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="p">Previous</button>
</a>
- <a rel="next" title="Entrypoint" href="entrypoint.html" >
+ <a rel="next" title="airflow.executors.local_executor" href="../local_executor/index.html" >
<button class="btn-hollow btn-blue bodytext__medium--cerulean-blue" accesskey="n">Next</button>
</a>
</div>
@@ -1300,30 +1265,117 @@ using COPY command if you want to get the files in your final image (in the main
<nav class="wy-nav-side-toc">
<div class="wy-menu-vertical">
<ul>
-<li><a class="reference internal" href="#">Building the image</a><ul>
-<li><a class="reference internal" href="#extending-vs-customizing-the-image">Extending vs. customizing the image</a></li>
-<li><a class="reference internal" href="#extending-the-image">Extending the image</a></li>
-<li><a class="reference internal" href="#examples-of-image-extending">Examples of image extending</a><ul>
-<li><a class="reference internal" href="#an-apt-package-example">An <code class="docutils literal notranslate"><span class="pre">apt</span></code> package example</a></li>
-<li><a class="reference internal" href="#a-pypi-package-example">A <code class="docutils literal notranslate"><span class="pre">PyPI</span></code> package example</a></li>
-<li><a class="reference internal" href="#a-umask-requiring-example">A <code class="docutils literal notranslate"><span class="pre">umask</span></code> requiring example</a></li>
-<li><a class="reference internal" href="#a-build-essential-requiring-package-example">A <code class="docutils literal notranslate"><span class="pre">build-essential</span></code> requiring package example</a></li>
-<li><a class="reference internal" href="#dag-embedding-example">DAG embedding example</a></li>
+<li><a class="reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">airflow.executors.kubernetes_executor</span></code></a><ul>
+<li><a class="reference internal" href="#module-contents">Module Contents</a><ul>
+<li><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobType">KubernetesJobType</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesResultsType">KubernetesResultsType</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesWatchType">KubernetesWatchType</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.kubernetes_executor.ResourceVersion">ResourceVersion</a><ul>
+<li><a class="reference internal" href="#airflow.executors.kubernetes_executor.ResourceVersion._instance">_instance</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.kubernetes_executor.ResourceVersion.resource_version">resource_version</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.kubernetes_executor.ResourceVersion.__new__">__new__</a></li>
+</ul>
+</li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobWatcher">KubernetesJobWatcher</a><ul>
+<li><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobWatcher.run">run</a></li>
+</ul>
+<ul>
+<li><a class="reference internal" href="#airflow.executors.kubernetes_executor.KubernetesJobWatcher._run">_run</a></li>
+</ul>
... 330392 lines suppressed ...