You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by GitBox <gi...@apache.org> on 2021/01/13 14:04:42 UTC
[GitHub] [airflow] mik-laj commented on issue #8605: Add Production-ready docker compose for the production image
mik-laj commented on issue #8605:
URL: https://github.com/apache/airflow/issues/8605#issuecomment-759469443
I have prepared some Dockerfiles with some common configuration.
<details>
<summary>Postgres - Redis - Airflow 2.0</summary>
```yaml
version: '3'
x-airflow-common:
&airflow-common
image: apache/airflow:2.0.0
environment:
- AIRFLOW__CORE__EXECUTOR=CeleryExecutor
- AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
- AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres/airflow
#- AIRFLOW__CELERY__RESULT_BACKEND=redis://:@redis:6379/0
- AIRFLOW__CELERY__BROKER_URL=redis://:@redis:6379/0
- AIRFLOW__WEBSERVER__RBAC=True
- AIRFLOW__CORE__FERNET_KEY=FB0o_zt4e3Ziq3LdUUO7F2Z95cvFFx16hU8jTeR1ASM=
- AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=True
volumes:
- ./dags:/opt/airflow/dags
- ./airflow-data/logs:/opt/airflow/logs
- ./airflow-data/plugins:/opt/airflow/plugins
depends_on:
redis:
condition: service_healthy
postgres:
condition: service_healthy
services:
postgres:
image: postgres:9.5
environment:
POSTGRES_USER: airflow
POSTGRES_PASSWORD: airflow
POSTGRES_DB: airflow
volumes:
- ./airflow-data/postgres-db-volume:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "pg_isready", "-U", "airflow"]
interval: 30s
retries: 5
restart: always
redis:
image: redis:latest
ports:
- 6379:6379
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 30s
retries: 50
restart: always
airflow-webserver:
<< : *airflow-common
command: webserver
ports:
- 8080:8080
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
interval: 10s
timeout: 10s
retries: 5
restart: always
airflow-scheduler:
<< : *airflow-common
command: scheduler
restart: always
airflow-worker:
<< : *airflow-common
command: celery worker
restart: always
airflow-init:
<< : *airflow-common
entrypoint: /bin/bash
command:
- -c
- airflow users list || (
airflow db init &&
airflow users create
--role Admin
--username airflow
--password airflow
--email airflow@airflow.com
--firstname airflow
--lastname airflow
)
restart: on-failure
flower:
<< : *airflow-common
command: celery flower
ports:
- 5555:5555
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:5555/"]
interval: 10s
timeout: 10s
retries: 5
restart: always
```
</details>
<details><summary>Postgres - Redis - Airflow 1.10.14</summary>
```yaml
version: '3'
x-airflow-common:
&airflow-common
image: apache/airflow:2.0.0
environment:
- AIRFLOW__CORE__EXECUTOR=CeleryExecutor
- AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
- AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres/airflow
- AIRFLOW__CELERY__BROKER_URL=redis://:@redis:6379/0
- AIRFLOW__CORE__FERNET_KEY=FB0o_zt4e3Ziq3LdUUO7F2Z95cvFFx16hU8jTeR1ASM=
- AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=True
volumes:
- ./dags:/opt/airflow/dags
- ./airflow-data/logs:/opt/airflow/logs
- ./airflow-data/plugins:/opt/airflow/plugins
depends_on:
redis:
condition: service_healthy
postgres:
condition: service_healthy
services:
postgres:
image: postgres:9.5
environment:
POSTGRES_USER: airflow
POSTGRES_PASSWORD: airflow
POSTGRES_DB: airflow
volumes:
- ./airflow-data/postgres-db-volume:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "pg_isready", "-U", "airflow"]
interval: 30s
retries: 5
restart: always
redis:
image: redis:latest
ports:
- 6379:6379
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 30s
retries: 50
restart: always
airflow-webserver:
<< : *airflow-common
command: webserver
ports:
- 8080:8080
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
interval: 10s
timeout: 10s
retries: 5
restart: always
airflow-scheduler:
<< : *airflow-common
command: scheduler
restart: always
airflow-worker:
<< : *airflow-common
command: worker
restart: always
airflow-init:
<< : *airflow-common
entrypoint: /bin/bash
command:
- -c
- airflow list_users || (
airflow initdb &&
airflow create_user
--role Admin
--username airflow
--password airflow
--email airflow@airflow.com
--firstname airflow
--lastname airflow
)
restart: on-failure
flower:
<< : *airflow-common
command: flower
ports:
- 5555:5555
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:5555/healthcheck"]
interval: 10s
timeout: 10s
retries: 5
restart: always
```
</details>
<details><summary>Mysql 8.0 - Redis - Airflow 2.0</summary>
```yaml
# Migrations are broken.
```
</details>
<details><summary>Mysql 8.0 - Redis - Airflow 1.10.14</summary>
```yaml
version: '3'
x-airflow-common:
&airflow-common
image: apache/airflow:1.10.14
environment:
- AIRFLOW__CORE__EXECUTOR=CeleryExecutor
- AIRFLOW__CORE__SQL_ALCHEMY_CONN=mysql://root:airflow@mysql/airflow?charset=utf8mb4
- AIRFLOW__CORE__SQL_ENGINE_COLLATION_FOR_IDS=utf8mb3_general_ci
- AIRFLOW__CELERY__BROKER_URL=redis://:@redis:6379/0
- AIRFLOW__CORE__FERNET_KEY=FB0o_zt4e3Ziq3LdUUO7F2Z95cvFFx16hU8jTeR1ASM=
- AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=True
volumes:
- ./dags:/opt/airflow/dags
- ./airflow-data/logs:/opt/airflow/logs
- ./airflow-data/plugins:/opt/airflow/plugins
depends_on:
redis:
condition: service_healthy
mysql:
condition: service_healthy
services:
mysql:
image: mysql:8.0
environment:
- MYSQL_ROOT_PASSWORD=airflow
- MYSQL_ROOT_HOST=%
- MYSQL_DATABASE=airflow
volumes:
- ./airflow-data/mysql-db-volume:/var/lib/mysql
ports:
- "3306:3306"
command:
- mysqld
- --explicit-defaults-for-timestamp
- --default-authentication-plugin=mysql_native_password
- --character-set-server=utf8mb4
- --collation-server=utf8mb4_unicode_ci
healthcheck:
test: ["CMD-SHELL", "mysql -h localhost -P 3306 -u root -pairflow -e 'SELECT 1'"]
interval: 10s
timeout: 10s
retries: 5
restart: always
redis:
image: redis:latest
ports:
- 6379:6379
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 30s
retries: 50
restart: always
airflow-webserver:
<< : *airflow-common
command: webserver
ports:
- 8080:8080
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
interval: 10s
timeout: 10s
retries: 5
restart: always
airflow-scheduler:
<< : *airflow-common
command: scheduler
restart: always
airflow-worker:
<< : *airflow-common
command: worker
restart: always
airflow-init:
<< : *airflow-common
entrypoint: /bin/bash
command:
- -c
- airflow list_users || (
airflow initdb &&
airflow create_user
--role Admin
--username airflow
--password airflow
--email airflow@airflow.com
--firstname airflow
--lastname airflow
)
restart: on-failure
flower:
<< : *airflow-common
command: flower
ports:
- 5555:5555
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:5555/"]
interval: 10s
timeout: 10s
retries: 5
restart: always
```
</details>
I added health checks where it was simple. Anyone have an idea for health-checks for `airflow-scheduler`/`airflow-worker`? This will improve stability.
Besides, I am planning to prepare a tool that is used to generate docker-compose files using a simple wizard. I am thinking of something similar to the Pytorch project.
https://pytorch.org/get-started/locally/
<img width="624" alt="Screenshot 2021-01-13 at 15 01 49" src="https://user-images.githubusercontent.com/12058428/104462017-55fa3c80-55b0-11eb-9e95-45e3cfb281ae.png">
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org