You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ma...@apache.org on 2024/04/12 23:07:55 UTC
(superset) branch master updated: feat: optimize docker-compose up for faster boot time (#27953)
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 40e77be813 feat: optimize docker-compose up for faster boot time (#27953)
40e77be813 is described below
commit 40e77be813c789c8b01aece739f32ff5753436b4
Author: Maxime Beauchemin <ma...@gmail.com>
AuthorDate: Fri Apr 12 16:07:49 2024 -0700
feat: optimize docker-compose up for faster boot time (#27953)
---
.github/workflows/docker.yml | 2 +-
.gitignore | 1 +
Dockerfile | 12 ++++--
docker-compose-image-tag.yml | 30 ++++++++++++---
docker-compose-non-dev.yml | 30 ++++++++++++---
docker-compose.yml | 44 +++++++++++++++++-----
docker/.env | 4 ++
docker/docker-bootstrap.sh | 8 ----
docker/docker-frontend.sh | 2 +-
.../installing-superset-using-docker-compose.mdx | 9 ++---
10 files changed, 105 insertions(+), 37 deletions(-)
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index e749ac35c8..849fb7b132 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -21,7 +21,7 @@ jobs:
steps:
- id: set_matrix
run: |
- MATRIX_CONFIG=$(if [ "${{ github.event_name }}" == "pull_request" ]; then echo '["ci"]'; else echo '["dev", "lean", "py310", "websocket", "dockerize"]'; fi)
+ MATRIX_CONFIG=$(if [ "${{ github.event_name }}" == "pull_request" ]; then echo '["dev"]'; else echo '["dev", "lean", "py310", "websocket", "dockerize"]'; fi)
echo "matrix_config=${MATRIX_CONFIG}" >> $GITHUB_OUTPUT
echo $GITHUB_OUTPUT
diff --git a/.gitignore b/.gitignore
index 02657eb0fa..bbfc640c9a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -110,6 +110,7 @@ release.json
messages.mo
docker/requirements-local.txt
+docker/.env-local
cache/
docker/*local*
diff --git a/Dockerfile b/Dockerfile
index 9946111383..6faaf180b9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -119,9 +119,15 @@ RUN apt-get update -qq \
libasound2 \
libxtst6 \
wget \
- pkg-config \
- # Install GeckoDriver WebDriver
- && wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
+ git \
+ pkg-config
+
+RUN pip install playwright
+RUN playwright install-deps
+RUN playwright install chromium
+
+# Install GeckoDriver WebDriver
+RUN wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
# Install Firefox
&& wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
&& ln -s /opt/firefox/firefox /usr/local/bin/firefox \
diff --git a/docker-compose-image-tag.yml b/docker-compose-image-tag.yml
index 07f0d0dcb1..6c017ef75e 100644
--- a/docker-compose-image-tag.yml
+++ b/docker-compose-image-tag.yml
@@ -33,7 +33,11 @@ services:
- redis:/data
db:
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
image: postgres:15
container_name: superset_db
restart: unless-stopped
@@ -42,7 +46,11 @@ services:
- ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
superset:
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
image: *superset-image
container_name: superset_app
command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
@@ -57,7 +65,11 @@ services:
image: *superset-image
container_name: superset_init
command: ["/app/docker/docker-init.sh"]
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
depends_on: *superset-depends-on
user: "root"
volumes: *superset-volumes
@@ -68,7 +80,11 @@ services:
image: *superset-image
container_name: superset_worker
command: ["/app/docker/docker-bootstrap.sh", "worker"]
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
restart: unless-stopped
depends_on: *superset-depends-on
user: "root"
@@ -84,7 +100,11 @@ services:
image: *superset-image
container_name: superset_worker_beat
command: ["/app/docker/docker-bootstrap.sh", "beat"]
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
restart: unless-stopped
depends_on: *superset-depends-on
user: "root"
diff --git a/docker-compose-non-dev.yml b/docker-compose-non-dev.yml
index f537e26c38..78d8cb9c35 100644
--- a/docker-compose-non-dev.yml
+++ b/docker-compose-non-dev.yml
@@ -38,7 +38,11 @@ services:
- redis:/data
db:
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
image: postgres:15
container_name: superset_db
restart: unless-stopped
@@ -47,7 +51,11 @@ services:
- ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
superset:
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
build:
<<: *common-build
container_name: superset_app
@@ -64,7 +72,11 @@ services:
build:
<<: *common-build
command: ["/app/docker/docker-init.sh"]
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
depends_on: *superset-depends-on
user: "root"
volumes: *superset-volumes
@@ -76,7 +88,11 @@ services:
<<: *common-build
container_name: superset_worker
command: ["/app/docker/docker-bootstrap.sh", "worker"]
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
restart: unless-stopped
depends_on: *superset-depends-on
user: "root"
@@ -93,7 +109,11 @@ services:
<<: *common-build
container_name: superset_worker_beat
command: ["/app/docker/docker-bootstrap.sh", "beat"]
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
restart: unless-stopped
depends_on: *superset-depends-on
user: "root"
diff --git a/docker-compose.yml b/docker-compose.yml
index 9252e59902..6822a90e05 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -54,7 +54,11 @@ services:
- redis:/data
db:
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
image: postgres:15
container_name: superset_db
restart: unless-stopped
@@ -65,7 +69,11 @@ services:
- ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
superset:
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
build:
<<: *common-build
container_name: superset_app
@@ -116,7 +124,11 @@ services:
<<: *common-build
container_name: superset_init
command: ["/app/docker/docker-init.sh"]
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
depends_on: *superset-depends-on
user: *superset-user
volumes: *superset-volumes
@@ -130,12 +142,14 @@ services:
environment:
# set this to false if you have perf issues running the npm i; npm run dev in-docker
# if you do so, you have to run this manually on the host, which should perform better!
- BUILD_SUPERSET_FRONTEND_IN_DOCKER: ${BUILD_SUPERSET_FRONTEND_IN_DOCKER:-true}
SCARF_ANALYTICS: "${SCARF_ANALYTICS}"
- PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: ${BUILD_SUPERSET_FRONTEND_IN_DOCKER:-false}
container_name: superset_node
command: ["/app/docker/docker-frontend.sh"]
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
depends_on: *superset-depends-on
volumes: *superset-volumes
@@ -144,7 +158,11 @@ services:
<<: *common-build
container_name: superset_worker
command: ["/app/docker/docker-bootstrap.sh", "worker"]
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
restart: unless-stopped
depends_on: *superset-depends-on
user: *superset-user
@@ -162,7 +180,11 @@ services:
<<: *common-build
container_name: superset_worker_beat
command: ["/app/docker/docker-bootstrap.sh", "beat"]
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
restart: unless-stopped
depends_on: *superset-depends-on
user: *superset-user
@@ -175,7 +197,11 @@ services:
<<: *common-build
container_name: superset_tests_worker
command: ["/app/docker/docker-bootstrap.sh", "worker"]
- env_file: docker/.env
+ env_file:
+ # defaults
+ - docker/.env
+ # gitignored overrides
+ - docker/.env-local
environment:
DATABASE_HOST: localhost
DATABASE_DB: test
diff --git a/docker/.env b/docker/.env
index 1b7d3df8c1..51b0c0d921 100644
--- a/docker/.env
+++ b/docker/.env
@@ -53,3 +53,7 @@ SUPERSET_PORT=8088
MAPBOX_API_KEY=''
SUPERSET_SECRET_KEY=TEST_NON_DEV_SECRET
+
+ENABLE_PLAYWRIGHT=false
+PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
+BUILD_SUPERSET_FRONTEND_IN_DOCKER=true
diff --git a/docker/docker-bootstrap.sh b/docker/docker-bootstrap.sh
index 7d53f92898..2f86342714 100755
--- a/docker/docker-bootstrap.sh
+++ b/docker/docker-bootstrap.sh
@@ -35,14 +35,6 @@ else
echo "Skipping local overrides"
fi
-#
-# playwright is an optional package - run only if it is installed
-#
-if command -v playwright > /dev/null 2>&1; then
- playwright install-deps
- playwright install chromium
-fi
-
case "${1}" in
worker)
echo "Starting Celery worker..."
diff --git a/docker/docker-frontend.sh b/docker/docker-frontend.sh
index 85c57cbf0f..c69fefd6d0 100755
--- a/docker/docker-frontend.sh
+++ b/docker/docker-frontend.sh
@@ -18,8 +18,8 @@
set -e
# Packages needed for puppeteer:
-apt update
if [ "$PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" = "false" ]; then
+ apt update
apt install -y chromium
fi
diff --git a/docs/docs/installation/installing-superset-using-docker-compose.mdx b/docs/docs/installation/installing-superset-using-docker-compose.mdx
index e41145fb09..4193a1f0fd 100644
--- a/docs/docs/installation/installing-superset-using-docker-compose.mdx
+++ b/docs/docs/installation/installing-superset-using-docker-compose.mdx
@@ -112,7 +112,6 @@ with docker compose will store that data in a PostgreSQL database contained in a
[volume](https://docs.docker.com/storage/volumes/), which is not backed up.
Again **DO NOT USE THIS FOR PRODUCTION**
-
:::
You should see a wall of logging output from the containers being launched on your machine. Once
@@ -127,10 +126,10 @@ can skip to the next section.
You can install additional python packages and apply config overrides by following the steps
mentioned in [docker/README.md](https://github.com/apache/superset/tree/master/docker#configuration)
-You can configure the Docker Compose environment variables for dev and non-dev mode with
-`docker/.env`. This environment file sets the environment
-for most containers in the Docker Compose setup, and some variables affect multiple containers and
-others only single ones.
+Note that `docker/.env` sets the default environment variables for all the docker images
+used by `docker-compose`, and that `docker/.env-local` can be used to override those defaults.
+Also note that `docker/.env-local` is referenced in our `.gitignore`,
+preventing developers from risking committing potentially sensitive configuration to the repository.
One important variable is `SUPERSET_LOAD_EXAMPLES` which determines whether the `superset_init`
container will populate example data and visualizations into the metadata database. These examples