You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by cr...@apache.org on 2023/06/24 18:42:08 UTC
[superset] branch master updated: build(docker): simplification of assembly to facilitate further support (#24504)
This is an automated email from the ASF dual-hosted git repository.
craigrueda pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new ba3bdc077c build(docker): simplification of assembly to facilitate further support (#24504)
ba3bdc077c is described below
commit ba3bdc077cfe1a017105332dc688b9d7faef6795
Author: Aleksey Karpov <86...@users.noreply.github.com>
AuthorDate: Sat Jun 24 21:41:57 2023 +0300
build(docker): simplification of assembly to facilitate further support (#24504)
---
.github/workflows/docker.yml | 2 +-
.github/workflows/docker_build_push.sh | 8 ++---
Dockerfile | 64 +++++++++++++---------------------
docker/docker-bootstrap.sh | 38 ++++++++++++--------
4 files changed, 52 insertions(+), 60 deletions(-)
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 7c5b50baf4..8a081a303c 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -50,7 +50,7 @@ jobs:
mkdir -p ./build
echo ${{ github.sha }} > ./build/SHA
echo ${{ github.event.pull_request.number }} > ./build/PR-NUM
- docker build --target ci -t ${{ github.sha }} -t "pr-${{ github.event.pull_request.number }}" .
+ DOCKER_BUILDKIT=1 docker build --target ci -t ${{ github.sha }} -t "pr-${{ github.event.pull_request.number }}" .
docker save ${{ github.sha }} | gzip > ./build/${{ github.sha }}.tar.gz
- name: Upload build artifacts
diff --git a/.github/workflows/docker_build_push.sh b/.github/workflows/docker_build_push.sh
index 55ccf51767..3ee615bc01 100755
--- a/.github/workflows/docker_build_push.sh
+++ b/.github/workflows/docker_build_push.sh
@@ -46,7 +46,7 @@ EOF
#
# Build the "lean" image
#
-docker build --target lean \
+DOCKER_BUILDKIT=1 docker build --target lean \
-t "${REPO_NAME}:${SHA}" \
-t "${REPO_NAME}:${REFSPEC}" \
-t "${REPO_NAME}:${LATEST_TAG}" \
@@ -59,7 +59,7 @@ docker build --target lean \
#
# Build the "lean310" image
#
-docker build --target lean \
+DOCKER_BUILDKIT=1 docker build --target lean \
-t "${REPO_NAME}:${SHA}-py310" \
-t "${REPO_NAME}:${REFSPEC}-py310" \
-t "${REPO_NAME}:${LATEST_TAG}-py310" \
@@ -73,7 +73,7 @@ docker build --target lean \
#
# Build the "websocket" image
#
-docker build \
+DOCKER_BUILDKIT=1 docker build \
-t "${REPO_NAME}:${SHA}-websocket" \
-t "${REPO_NAME}:${REFSPEC}-websocket" \
-t "${REPO_NAME}:${LATEST_TAG}-websocket" \
@@ -86,7 +86,7 @@ docker build \
#
# Build the dev image
#
-docker build --target dev \
+DOCKER_BUILDKIT=1 docker build --target dev \
-t "${REPO_NAME}:${SHA}-dev" \
-t "${REPO_NAME}:${REFSPEC}-dev" \
-t "${REPO_NAME}:${LATEST_TAG}-dev" \
diff --git a/Dockerfile b/Dockerfile
index 494bf90bd1..a5ee4e0a71 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -29,17 +29,17 @@ ENV BUILD_CMD=${NPM_BUILD_CMD}
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
# NPM ci first, as to NOT invalidate previous steps except for when package.json changes
-RUN mkdir -p /app/superset-frontend
+WORKDIR /app/superset-frontend
COPY ./docker/frontend-mem-nag.sh /
-RUN /frontend-mem-nag.sh
-WORKDIR /app/superset-frontend/
+RUN /frontend-mem-nag.sh
COPY superset-frontend/package*.json ./
+
RUN npm ci
-COPY ./superset-frontend .
+COPY ./superset-frontend ./
# This seems to be the most expensive step
RUN npm run ${BUILD_CMD}
@@ -49,6 +49,7 @@ RUN npm run ${BUILD_CMD}
######################################################################
FROM python:${PY_VER} AS lean
+WORKDIR /app
ENV LANG=C.UTF-8 \
LC_ALL=C.UTF-8 \
FLASK_ENV=production \
@@ -59,8 +60,8 @@ ENV LANG=C.UTF-8 \
RUN mkdir -p ${PYTHONPATH} \
&& useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \
- && apt-get update -y \
- && apt-get install -y --no-install-recommends \
+ && apt-get update -q \
+ && apt-get install -yq --no-install-recommends \
build-essential \
curl \
default-libmysqlclient-dev \
@@ -71,11 +72,8 @@ RUN mkdir -p ${PYTHONPATH} \
libldap2-dev \
&& rm -rf /var/lib/apt/lists/*
-WORKDIR /app
-
COPY --chown=superset:superset ./requirements/*.txt requirements/
COPY --chown=superset:superset setup.py MANIFEST.in README.md ./
-
# setup.py uses the version information in package.json
COPY --chown=superset:superset superset-frontend/package.json superset-frontend/
@@ -84,16 +82,14 @@ RUN mkdir -p superset/static \
&& pip install --no-cache-dir -r requirements/local.txt
COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
-
## Lastly, let's install superset itself
COPY --chown=superset:superset superset superset
+
RUN chown -R superset:superset ./* \
&& pip install --no-cache-dir -e . \
&& flask fab babel-compile --target superset/translations
-COPY ./docker/run-server.sh /usr/bin/
-RUN chmod a+x /usr/bin/run-server.sh
-
+COPY --chmod=755 ./docker/run-server.sh /usr/bin/
USER superset
HEALTHCHECK CMD curl -f "http://localhost:$SUPERSET_PORT/health"
@@ -109,47 +105,35 @@ FROM lean AS dev
ARG GECKODRIVER_VERSION=v0.32.0
ARG FIREFOX_VERSION=106.0.3
-COPY ./requirements/*.txt ./docker/requirements-*.txt/ /app/requirements/
-
USER root
-RUN apt-get update -y \
- && apt-get install -y --no-install-recommends \
+RUN apt-get update -q \
+ && apt-get install -yq --no-install-recommends \
libnss3 \
libdbus-glib-1-2 \
libgtk-3-0 \
libx11-xcb1 \
libasound2 \
libxtst6 \
- wget
-
-# Install GeckoDriver WebDriver
-RUN wget https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O /tmp/geckodriver.tar.gz && \
- tar xvfz /tmp/geckodriver.tar.gz -C /tmp && \
- mv /tmp/geckodriver /usr/local/bin/geckodriver && \
- rm /tmp/geckodriver.tar.gz
-
-# Install Firefox
-RUN wget https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O /opt/firefox.tar.bz2 && \
- tar xvf /opt/firefox.tar.bz2 -C /opt && \
- ln -s /opt/firefox/firefox /usr/local/bin/firefox
+ wget \
+ # Install GeckoDriver WebDriver
+ && wget https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
+ # Install Firefox
+ && wget https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
+ && ln -s /opt/firefox/firefox /usr/local/bin/firefox \
+ && apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/* && apt-get clean
+COPY ./requirements/*.txt ./docker/requirements-*.txt/ /app/requirements/
# Cache everything for dev purposes...
-RUN cd /app \
- && pip install --no-cache -r requirements/docker.txt \
- && pip install --no-cache -r requirements/requirements-local.txt || true
-USER superset
-
+RUN pip install --no-cache-dir -r /app/requirements/docker.txt \
+ && pip install --no-cache-dir -r /app/requirements/requirements-local.txt || true
+USER superset
######################################################################
# CI image...
######################################################################
FROM lean AS ci
-COPY --chown=superset ./docker/docker-bootstrap.sh /app/docker/
-COPY --chown=superset ./docker/docker-init.sh /app/docker/
-COPY --chown=superset ./docker/docker-ci.sh /app/docker/
-
-RUN chmod a+x /app/docker/*.sh
+COPY --chown=superset --chmod=755 ./docker/*.sh /app/docker/
-CMD /app/docker/docker-ci.sh
+CMD ["/app/docker/docker-ci.sh"]
diff --git a/docker/docker-bootstrap.sh b/docker/docker-bootstrap.sh
index 6b92e97002..ad6ee57616 100755
--- a/docker/docker-bootstrap.sh
+++ b/docker/docker-bootstrap.sh
@@ -30,22 +30,30 @@ fi
#
if [ -f "${REQUIREMENTS_LOCAL}" ]; then
echo "Installing local overrides at ${REQUIREMENTS_LOCAL}"
- pip install -r "${REQUIREMENTS_LOCAL}"
+ pip install --no-cache-dir -r "${REQUIREMENTS_LOCAL}"
else
echo "Skipping local overrides"
fi
-if [[ "${1}" == "worker" ]]; then
- echo "Starting Celery worker..."
- celery --app=superset.tasks.celery_app:app worker -O fair -l INFO
-elif [[ "${1}" == "beat" ]]; then
- echo "Starting Celery beat..."
- rm -f /tmp/celerybeat.pid
- celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule
-elif [[ "${1}" == "app" ]]; then
- echo "Starting web app..."
- flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
-elif [[ "${1}" == "app-gunicorn" ]]; then
- echo "Starting web app..."
- /usr/bin/run-server.sh
-fi
+case "${1}" in
+ worker)
+ echo "Starting Celery worker..."
+ celery --app=superset.tasks.celery_app:app worker -O fair -l INFO
+ ;;
+ beat)
+ echo "Starting Celery beat..."
+ rm -f /tmp/celerybeat.pid
+ celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule
+ ;;
+ app)
+ echo "Starting web app..."
+ flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
+ ;;
+ app-gunicorn)
+ echo "Starting web app..."
+ /usr/bin/run-server.sh
+ ;;
+ *)
+ echo "Unknown Operation!!!"
+ ;;
+esac