You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ma...@apache.org on 2024/02/15 06:34:21 UTC
(superset) 01/01: feat: faster docker builds
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch faster_docker
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 89b7dd1c1a3dacf72e1159f7708c68a5b6574d5c
Author: Maxime Beauchemin <ma...@gmail.com>
AuthorDate: Fri Feb 9 10:01:43 2024 -0800
feat: faster docker builds
---
Dockerfile | 30 ++++++++++++++++++++-------
scripts/get_package_version.py | 11 ++++++++++
setup.py | 46 ++++++++++++++++++++++++++++--------------
version.txt | 1 +
4 files changed, 66 insertions(+), 22 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index fc3e667037..0f24bdbba9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -47,11 +47,24 @@ COPY ./superset-frontend ./
# This seems to be the most expensive step
RUN npm run ${BUILD_CMD}
+######################################################################
+# Python pre-lean image
+######################################################################
+FROM python:${PY_VER} AS pre_lean
+WORKDIR /app
+RUN mkdir scripts ./superset-frontend/
+COPY --chown=superset:superset ./scripts/get_package_version.py ./scripts
+COPY --chown=superset:superset ./superset-frontend/package.json ./superset-frontend/
+# Generate ./version.txt from package.json (the source of thruth)
+RUN python ./scripts/get_package_version.py > ./version.txt
+
+
######################################################################
# Final lean image...
######################################################################
FROM python:${PY_VER} AS lean
+
WORKDIR /app
ENV LANG=C.UTF-8 \
LC_ALL=C.UTF-8 \
@@ -61,9 +74,7 @@ ENV LANG=C.UTF-8 \
SUPERSET_HOME="/app/superset_home" \
SUPERSET_PORT=8088
-RUN mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg-info requirements \
- && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \
- && apt-get update -qq && apt-get install -yqq --no-install-recommends \
+RUN apt-get update -qq && apt-get install -yqq --no-install-recommends \
build-essential \
curl \
default-libmysqlclient-dev \
@@ -72,13 +83,18 @@ RUN mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg
libpq-dev \
libecpg-dev \
libldap2-dev \
- && touch superset/static/version_info.json \
- && chown -R superset:superset ./* \
&& rm -rf /var/lib/apt/lists/*
+RUN mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg-info requirements \
+ && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \
+ && touch superset/static/version_info.json \
+ && chown -R superset:superset ./*
+
COPY --chown=superset:superset setup.py MANIFEST.in README.md ./
-# setup.py uses the version information in package.json
-COPY --chown=superset:superset superset-frontend/package.json superset-frontend/
+
+# version_info.json is used by setup.py
+COPY --chown=superset:superset --from=pre_lean /app/version.txt ./
+
RUN --mount=type=bind,target=./requirements/local.txt,src=./requirements/local.txt \
--mount=type=bind,target=./requirements/development.txt,src=./requirements/development.txt \
--mount=type=bind,target=./requirements/base.txt,src=./requirements/base.txt \
diff --git a/scripts/get_package_version.py b/scripts/get_package_version.py
new file mode 100644
index 0000000000..60b5723569
--- /dev/null
+++ b/scripts/get_package_version.py
@@ -0,0 +1,11 @@
+import json
+import os
+
+BASE_DIR = os.path.abspath(os.path.dirname(__file__))
+PACKAGE_JSON = os.path.join(BASE_DIR, "../", "superset-frontend", "package.json")
+
+
+if os.path.exists(PACKAGE_JSON):
+ # package.json is the source of truth for version info
+ with open(PACKAGE_JSON) as f:
+ print(json.load(f)["version"])
diff --git a/setup.py b/setup.py
index 97df81a695..7428ac2710 100644
--- a/setup.py
+++ b/setup.py
@@ -22,12 +22,21 @@ from setuptools import find_packages, setup
BASE_DIR = os.path.abspath(os.path.dirname(__file__))
PACKAGE_JSON = os.path.join(BASE_DIR, "superset-frontend", "package.json")
+VERSION_INFO_FILE = os.path.join(BASE_DIR, "superset", "static", "version_info.json")
+DOCKER_VERSION_FILE = os.path.join(BASE_DIR, "version.txt")
-with open(PACKAGE_JSON) as package_file:
- version_string = json.load(package_file)["version"]
-with open("README.md", encoding="utf-8") as f:
- long_description = f.read()
+def get_version() -> str:
+ version = ""
+ if os.path.exists(PACKAGE_JSON):
+ # package.json is the source of truth
+ with open(PACKAGE_JSON) as f:
+ version = json.load(f)["version"]
+ elif os.path.exists(DOCKER_VERSION_FILE):
+ # to improve docker caching, we prepare a small version.txt
+ with open(DOCKER_VERSION_FILE) as f:
+ version = f.read().strip()
+ return version
def get_git_sha() -> str:
@@ -38,24 +47,31 @@ def get_git_sha() -> str:
return ""
-GIT_SHA = get_git_sha()
-version_info = {"GIT_SHA": GIT_SHA, "version": version_string}
-print("-==-" * 15)
-print("VERSION: " + version_string)
-print("GIT SHA: " + GIT_SHA)
-print("-==-" * 15)
+def stamp_version() -> None:
+ """Leaving a trace in stdout and static assets"""
+ GIT_SHA = get_git_sha()
+ version_info = {"GIT_SHA": GIT_SHA, "version": VERSION}
+ print("-==-" * 15)
+ print(f"VERSION: {VERSION}")
+ print(f"GIT SHA: {GIT_SHA}")
+ print("-==-" * 15)
-VERSION_INFO_FILE = os.path.join(BASE_DIR, "superset", "static", "version_info.json")
+ with open(VERSION_INFO_FILE, "w") as version_file:
+ json.dump(version_info, version_file)
-with open(VERSION_INFO_FILE, "w") as version_file:
- json.dump(version_info, version_file)
+
+VERSION = get_version()
+stamp_version()
+
+with open("README.md", encoding="utf-8") as f:
+ long_description = f.read()
setup(
name="apache-superset",
description="A modern, enterprise-ready business intelligence web application",
long_description=long_description,
long_description_content_type="text/markdown",
- version=version_string,
+ version=VERSION,
packages=find_packages(),
include_package_data=True,
zip_safe=False,
@@ -211,7 +227,7 @@ setup(
author="Apache Software Foundation",
author_email="dev@superset.apache.org",
url="https://superset.apache.org/",
- download_url="https://www.apache.org/dist/superset/" + version_string,
+ download_url="https://www.apache.org/dist/superset/" + VERSION,
classifiers=[
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
diff --git a/version.txt b/version.txt
new file mode 100644
index 0000000000..cb676de104
--- /dev/null
+++ b/version.txt
@@ -0,0 +1 @@
+0.0.0-dev