You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by vi...@apache.org on 2026/09/21 20:04:05 UTC
(superset) branch master updated: fix(ci): align Docker image publishing policy (#44250)
This is an automated email from the ASF dual-hosted git repository.
villebro pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 322eaeaf991 fix(ci): align Docker image publishing policy (#44250)
322eaeaf991 is described below
commit 322eaeaf99172f1f50d0aaeb7feea9e14c4b05af
Author: Ville Brofeldt <33...@users.noreply.github.com>
AuthorDate: Mon Sep 21 13:03:57 2026 -0700
fix(ci): align Docker image publishing policy (#44250)
---
.github/workflows/docker.yml | 299 ++++++++++++++++++---
.github/workflows/frontend-bundle-size-nightly.yml | 2 +-
.../workflows/scheduled-docker-image-refresh.yml | 195 --------------
.github/workflows/superset-frontend.yml | 2 +-
.github/workflows/tag-release.yml | 22 +-
UPDATING.md | 16 ++
docker-compose-light.yml | 2 +-
docker-compose-non-dev.yml | 2 +-
docker-compose.yml | 2 +-
docs/admin_docs/installation/docker-builds.mdx | 66 ++---
docs/admin_docs/installation/docker-compose.mdx | 2 +-
scripts/change_detector.py | 1 -
scripts/docker-build-extra-flags.sh | 49 +++-
scripts/docker-build-plan.sh | 50 ++++
scripts/docker-publish-guard.sh | 53 ++++
.../scheduled_docker_image_refresh_test.py | 65 -----
tests/unit_tests/scripts/docker_cache_ref_test.py | 90 +++++++
17 files changed, 576 insertions(+), 342 deletions(-)
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 6e92744c553..5a7cc104f78 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -13,10 +13,6 @@ permissions:
contents: read
pull-requests: read
-concurrency:
- group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
- cancel-in-progress: true
-
jobs:
changes:
runs-on: ubuntu-26.04
@@ -47,19 +43,18 @@ jobs:
steps:
- id: set_matrix
run: |
- MATRIX_CONFIG=$(if [ "${{ github.event_name }}" == "pull_request" ]; then echo '["superset", "dev", "lean"]'; else echo '["superset", "dev", "lean", "websocket", "dockerize", "py311", "py312"]'; fi)
+ MATRIX_CONFIG=$(if [ "${{ github.event_name }}" == "pull_request" ]; then echo '["superset", "dev", "lean"]'; else echo '["superset", "dev", "lean", "py311", "py312"]'; fi)
echo "matrix_config=${MATRIX_CONFIG}" >> $GITHUB_OUTPUT
echo $GITHUB_OUTPUT
# Runs unconditionally (no dependency on `changes`, and no build-preset
- # matrix restriction) so a regression in the PY_VER override logic is
- # always caught on PRs. Without this, the real docker-build job only runs
- # when the change detector flags docker/python/frontend changes (a
+ # matrix restriction) so regressions in the assembled Supersetbot command
+ # are always caught on PRs. Without this, the real docker-build job only
+ # runs when the change detector flags docker/python/frontend changes (a
# workflow-only edit like this one does not), and even then the PR build
- # matrix never includes the "py311"/"py312" presets that logic protects -
- # so a break here would otherwise first surface on a push to master.
- pyver-override-check:
- name: verify docker build PY_VER override
+ # matrix never includes the "py311"/"py312" presets that logic protects.
+ docker-command-check:
+ name: verify docker build command
runs-on: ubuntu-26.04
timeout-minutes: 5
steps:
@@ -69,7 +64,7 @@ jobs:
persist-credentials: false
- name: Setup supersetbot
uses: $/.github/actions/setup-supersetbot/
- - name: Assert PY_VER override applies to every preset except py311/py312
+ - name: Assert generated docker build command
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -80,10 +75,51 @@ jobs:
# so a regression in supersetbot itself (dropping the py311/py312
# PY_VER pin, or reordering args so our override no longer lands
# last) is caught here too, instead of only surfacing on master.
+ docker_command() {
+ local preset="$1" extra_flags="$2"
+ shift 2
+ # Supersetbot adds its own --cache-to whenever DOCKERHUB_TOKEN is
+ # present. Hide the token from Supersetbot so cache export is
+ # controlled only by docker-build-extra-flags.sh, mirroring the
+ # docker-build job. assert_supersetbot_exports_cache_with_token
+ # below covers the unhidden case.
+ DOCKERHUB_TOKEN= supersetbot docker \
+ --preset "$preset" \
+ --platform linux/amd64 \
+ "$@" \
+ --extra-flags "$extra_flags" \
+ --dry-run
+ }
+ docker_command_with_token() {
+ local preset="$1" extra_flags="$2"
+ shift 2
+ DOCKERHUB_TOKEN=dummy supersetbot docker \
+ --preset "$preset" \
+ --platform linux/amd64 \
+ "$@" \
+ --extra-flags "$extra_flags" \
+ --dry-run
+ }
+ load_build_plan() {
+ local event_name="$1" ref="$2" ref_name="$3"
+ eval "$(bash scripts/docker-build-plan.sh "$event_name" "$ref" "$ref_name")"
+ }
+ assert_build_plan() {
+ local event_name="$1" ref="$2" ref_name="$3" expected_context="$4" expected_context_ref="$5" expected_push_or_load="$6" expected_platform="$7" expected_cache_publish="$8"
+ load_build_plan "$event_name" "$ref" "$ref_name"
+ if [ "$BUILD_CONTEXT" != "$expected_context" ] ||
+ [ "$BUILD_CONTEXT_REF" != "$expected_context_ref" ] ||
+ [ "$PUSH_OR_LOAD" != "$expected_push_or_load" ] ||
+ [ "$PLATFORM_ARG" != "$expected_platform" ] ||
+ [ "$PUBLISH_DOCKER_CACHE" != "$expected_cache_publish" ]; then
+ echo "::error::unexpected docker build plan for event=$event_name ref=$ref ref_name=$ref_name: context=$BUILD_CONTEXT context_ref=$BUILD_CONTEXT_REF push_or_load=$PUSH_OR_LOAD platform=$PLATFORM_ARG publish_cache=$PUBLISH_DOCKER_CACHE"
+ exit 1
+ fi
+ }
assert_effective_py_ver() {
local preset="$1" expected="$2" extra_flags command actual
extra_flags="$(scripts/docker-build-extra-flags.sh "$preset" dummy-tag)"
- command="$(supersetbot docker --preset "$preset" --platform linux/amd64 --extra-flags "$extra_flags" --dry-run)"
+ command="$(docker_command "$preset" "$extra_flags")"
# docker buildx keeps the LAST value of a repeated --build-arg key.
actual="$(grep -oE -- '--build-arg PY_VER=[^[:space:]]+' <<<"$command" | tail -1)"
if [ "$actual" != "--build-arg PY_VER=$expected" ]; then
@@ -91,12 +127,155 @@ jobs:
exit 1
fi
}
- for preset in superset dev lean websocket dockerize; do
+ assert_cache_ref() {
+ local preset="$1" expected="$2" extra_flags command
+ extra_flags="$(scripts/docker-build-extra-flags.sh "$preset" dummy-tag)"
+ command="$(docker_command "$preset" "$extra_flags")"
+ if ! grep -Fq -- "--cache-from=type=registry,ref=apache/superset-cache:$expected" <<<"$command"; then
+ echo "::error::preset '$preset' expected cache-from apache/superset-cache:$expected (full command: $command)"
+ exit 1
+ fi
+ }
+ assert_master_tag() {
+ local preset="$1" expected="$2" extra_flags command
+ load_build_plan push refs/heads/master master
+ extra_flags="$(scripts/docker-build-extra-flags.sh "$preset" dummy-tag)"
+ command="$(docker_command "$preset" "$extra_flags" --context "$BUILD_CONTEXT" --context-ref "$BUILD_CONTEXT_REF")"
+ if ! grep -Eq -- "-t apache/superset:${expected}([[:space:]]|$)" <<<"$command"; then
+ echo "::error::preset '$preset' expected master tag apache/superset:$expected (full command: $command)"
+ exit 1
+ fi
+ }
+ assert_no_cache_push_for_validation() {
+ local preset="$1" extra_flags command
+ load_build_plan push refs/heads/7.0 7.0
+ extra_flags="$(PUBLISH_DOCKER_CACHE="" DOCKERHUB_TOKEN=dummy scripts/docker-build-extra-flags.sh "$preset" dummy-tag)"
+ command="$(GITHUB_REF=refs/heads/7.0 docker_command "$preset" "$extra_flags" --context "$BUILD_CONTEXT" --context-ref "$BUILD_CONTEXT_REF")"
+ if grep -Fq -- "--cache-to=type=registry" <<<"$command"; then
+ echo "::error::preset '$preset' unexpectedly exports registry cache during validation (full command: $command)"
+ exit 1
+ fi
+ }
+ # The assertion above only proves our own helper stays quiet; it is
+ # the docker-build job blanking DOCKERHUB_TOKEN that stops Supersetbot
+ # from adding an implicit --cache-to of its own. Pin that Supersetbot
+ # behaviour here, so the reason the production guard exists cannot
+ # quietly disappear and leave validation builds exporting unreviewed
+ # layers to the shared registry cache.
+ assert_supersetbot_exports_cache_with_token() {
+ local preset="$1" extra_flags command
+ load_build_plan push refs/heads/7.0 7.0
+ extra_flags="$(PUBLISH_DOCKER_CACHE="" DOCKERHUB_TOKEN=dummy scripts/docker-build-extra-flags.sh "$preset" dummy-tag)"
+ command="$(GITHUB_REF=refs/heads/7.0 docker_command_with_token "$preset" "$extra_flags" --context "$BUILD_CONTEXT" --context-ref "$BUILD_CONTEXT_REF")"
+ if ! grep -Fq -- "--cache-to=type=registry" <<<"$command"; then
+ echo "::error::supersetbot no longer adds an implicit --cache-to when DOCKERHUB_TOKEN is set; the token-hiding guard in docker-build may be obsolete (full command: $command)"
+ exit 1
+ fi
+ }
+ assert_publish_guard() {
+ local event_name="$1" ref="$2" sha="$3" tip_sha="$4" expected="$5"
+ eval "$(bash scripts/docker-publish-guard.sh "$event_name" "$ref" "$sha" "$tip_sha")"
+ if [ "$PUBLISH" != "$expected" ]; then
+ echo "::error::publish guard for event=$event_name ref=$ref sha=$sha tip=$tip_sha expected PUBLISH=$expected, got $PUBLISH"
+ exit 1
+ fi
+ }
+ assert_cache_push_for_master() {
+ local preset="$1" expected="$2" extra_flags command
+ load_build_plan push refs/heads/master master
+ extra_flags="$(PUBLISH_DOCKER_CACHE=true DOCKERHUB_TOKEN=dummy scripts/docker-build-extra-flags.sh "$preset" dummy-tag)"
+ command="$(docker_command "$preset" "$extra_flags" --context "$BUILD_CONTEXT" --context-ref "$BUILD_CONTEXT_REF")"
+ if ! grep -Fq -- "--cache-to=type=registry,mode=max,ref=apache/superset-cache:$expected" <<<"$command"; then
+ echo "::error::preset '$preset' expected cache-to apache/superset-cache:$expected for master publish (full command: $command)"
+ exit 1
+ fi
+ }
+ assert_release_tag() {
+ local preset="$1" expected="$2" extra_flags command
+ extra_flags="$(scripts/docker-build-extra-flags.sh "$preset" "" release)"
+ command="$(docker_command "$preset" "$extra_flags" --context release --context-ref 6.1.0)"
+ if ! grep -Eq -- "-t apache/superset:${expected}([[:space:]]|$)" <<<"$command"; then
+ echo "::error::preset '$preset' expected release tag apache/superset:$expected (full command: $command)"
+ exit 1
+ fi
+ if grep -Fq -- "--build-arg INCLUDE_CHROMIUM=" <<<"$command"; then
+ echo "::error::preset '$preset' overrides the release Dockerfile's Chromium default (full command: $command)"
+ exit 1
+ fi
+ }
+ # tag-release.yml snapshots this helper from master, then checks out
+ # the release ref before running it, so the helper has to read the
+ # Dockerfile from the working tree rather than from wherever it
+ # lives. Simulate that ref swap against a synthetic older tree: a
+ # real release ref pins an older PY_VER and carries its own Chromium
+ # default, neither of which release mode may override.
+ assert_release_ref_swap() {
+ local snapshot release_tree flags
+ snapshot="$(mktemp -d)/docker-build-extra-flags.sh"
+ cp scripts/docker-build-extra-flags.sh "$snapshot"
+ release_tree="$(mktemp -d)"
+ printf 'ARG PY_VER=3.11.13-slim-bookworm\n' >"$release_tree/Dockerfile"
+ flags="$(cd "$release_tree" && bash "$snapshot" superset "" release)"
+ if ! grep -Fq -- "--build-arg PY_VER=3.11.13-slim-bookworm" <<<"$flags"; then
+ echo "::error::release build ignored the checked-out Dockerfile's PY_VER (flags: $flags)"
+ exit 1
+ fi
+ if grep -Fq -- "--build-arg INCLUDE_CHROMIUM=" <<<"$flags"; then
+ echo "::error::release build overrode the checked-out Dockerfile's Chromium default (flags: $flags)"
+ exit 1
+ fi
+ # The same tree in ci mode must still get the validation override,
+ # so the two modes cannot silently converge.
+ flags="$(cd "$release_tree" && bash "$snapshot" superset dummy-tag ci)"
+ if ! grep -Fq -- "--build-arg INCLUDE_CHROMIUM=false" <<<"$flags"; then
+ echo "::error::ci build lost the INCLUDE_CHROMIUM=false override (flags: $flags)"
+ exit 1
+ fi
+ }
+ assert_build_plan push refs/heads/master master push master --push "--platform linux/arm64 --platform linux/amd64" true
+ assert_build_plan push refs/heads/7.0 7.0 push "" --load "--platform linux/amd64" ""
+ assert_build_plan pull_request refs/pull/44250/merge "" pull_request "" --load "--platform linux/amd64" ""
+ # A master push publishes only while its commit is still the branch
+ # tip. The second case is the delayed-A/fast-B inversion: A's change
+ # detection lagged, B published first, and A must not roll the
+ # mutable master* tags back to the older digest.
+ assert_publish_guard push refs/heads/master aaaa111 aaaa111 true
+ assert_publish_guard push refs/heads/master aaaa111 bbbb222 false
+ assert_publish_guard push refs/heads/7.0 aaaa111 bbbb222 true
+ assert_publish_guard pull_request refs/pull/44250/merge aaaa111 bbbb222 true
+ assert_supersetbot_exports_cache_with_token superset
+ for preset in superset dev lean; do
assert_effective_py_ver "$preset" "3.11.14-slim-trixie"
+ expected_cache="3.11.14-slim-trixie"
+ if [ "$preset" != "superset" ]; then
+ expected_cache="${expected_cache}-${preset}"
+ fi
+ assert_cache_ref "$preset" "$expected_cache"
+ assert_no_cache_push_for_validation "$preset"
done
assert_effective_py_ver py311 "3.11-slim-bookworm"
+ assert_cache_ref py311 "3.11-slim-bookworm"
+ assert_no_cache_push_for_validation py311
assert_effective_py_ver py312 "3.12-slim-bookworm"
- echo "PY_VER override logic verified against the assembled buildx command for all build presets"
+ assert_cache_ref py312 "3.12-slim-bookworm"
+ assert_no_cache_push_for_validation py312
+ assert_master_tag superset master
+ assert_master_tag dev master-dev
+ assert_master_tag lean master-lean
+ assert_master_tag py311 master-py311
+ assert_master_tag py312 master-py312
+ assert_cache_push_for_master superset "3.11.14-slim-trixie"
+ assert_cache_push_for_master dev "3.11.14-slim-trixie-dev"
+ assert_cache_push_for_master lean "3.11.14-slim-trixie-lean"
+ assert_cache_push_for_master py311 "3.11-slim-bookworm"
+ assert_cache_push_for_master py312 "3.12-slim-bookworm"
+ assert_release_tag superset "6.1.0"
+ assert_release_tag dev "6.1.0-dev"
+ assert_release_tag lean "6.1.0-lean"
+ assert_release_tag py311 "6.1.0-py311"
+ assert_release_tag py312 "6.1.0-py312"
+ assert_release_ref_swap
+ echo "Docker build command verified for Python versions, cache refs, cache export, publish guard, master tags, release tags, and the release ref swap"
docker-build:
name: docker-build
@@ -117,6 +296,15 @@ jobs:
matrix:
build_preset: ${{fromJson(needs.setup_matrix.outputs.matrix_config)}}
fail-fast: false
+ # Apply concurrency after change detection. A docs-only push has no
+ # docker-build job and cannot replace a pending publisher. Publishing
+ # pushes queue so a slower run cannot overwrite mutable tags, while
+ # superseded pull_request validation builds are cancelled as before --
+ # otherwise every new commit on a PR would wait behind the abandoned
+ # merge ref's build for up to the 60 minute job timeout.
+ concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build_preset }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -156,21 +344,55 @@ jobs:
- name: Setup supersetbot
uses: $/.github/actions/setup-supersetbot/
+ # Concurrency serializes the master matrix per preset, but it does not
+ # order the runs entering it: a run whose change detection lagged can
+ # acquire the lock after a newer commit already published, and would
+ # then move the mutable `master*` tags back to the older digest. Reject
+ # the stale SHA instead. Every later step is gated on the result,
+ # because a skipped build leaves no $IMAGE_TAG for them to act on.
+ - name: Guard against publishing a superseded master commit
+ id: publish_guard
+ shell: bash
+ run: |
+ set -euo pipefail
+ TIP_SHA=""
+ if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF" = "refs/heads/master" ]; then
+ # Public repo, so ls-remote needs no credentials; this keeps
+ # persist-credentials: false on the checkout above. Retried
+ # because a transient blip here would otherwise fail the job
+ # rather than let it publish from a possibly superseded commit.
+ for attempt in 1 2 3; do
+ if TIP_SHA="$(git ls-remote "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" refs/heads/master | cut -f1)" &&
+ [ -n "$TIP_SHA" ]; then
+ break
+ fi
+ if [ "$attempt" -eq 3 ]; then
+ echo "::error::could not read the master tip to validate $GITHUB_SHA"
+ exit 1
+ fi
+ echo "::warning::ls-remote attempt ${attempt} failed; retrying in 10s..."
+ sleep 10
+ done
+ fi
+ eval "$(bash scripts/docker-publish-guard.sh "$GITHUB_EVENT_NAME" "$GITHUB_REF" "$GITHUB_SHA" "$TIP_SHA")"
+ if [ "$PUBLISH" != "true" ]; then
+ echo "::notice::master advanced to ${TIP_SHA} while ${GITHUB_SHA} was queued; skipping this build"
+ fi
+ echo "publish=$PUBLISH" >> "$GITHUB_OUTPUT"
+
- name: Build Docker Image
+ if: steps.publish_guard.outputs.publish == 'true'
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BUILD_PRESET: ${{ matrix.build_preset }}
run: |
- # Single platform builds in pull_request context to speed things up
- if [ "$GITHUB_EVENT_NAME" = "push" ]; then
- PLATFORM_ARG="--platform linux/arm64 --platform linux/amd64"
- # can only --load images in single-platform builds
- PUSH_OR_LOAD="--push"
- elif [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
- PLATFORM_ARG="--platform linux/amd64"
- PUSH_OR_LOAD="--load"
- fi
+ set -euo pipefail
+ # Release branch pushes and pull requests validate the build
+ # locally. Official release images are published only by the
+ # release workflow after the vote completes.
+ eval "$(bash scripts/docker-build-plan.sh "$GITHUB_EVENT_NAME" "$GITHUB_REF" "${GITHUB_REF_NAME:-}")"
+ export PUBLISH_DOCKER_CACHE
# Retry to absorb transient Docker Hub registry errors (base-image
# pull timeouts, 504/401 on push, ECONNRESET) that otherwise fail
@@ -180,17 +402,21 @@ jobs:
# See scripts/docker-build-extra-flags.sh for why "py311"/"py312"
# are excluded from the PY_VER override applied to every other
# preset; that logic is also exercised on every PR by the
- # always-on pyver-override-check job below, since this job itself
+ # always-on docker-command-check job, since this job itself
# only runs when the change detector flags docker/python/frontend
# changes and the PR build matrix never includes py311/py312.
EXTRA_FLAGS="$(scripts/docker-build-extra-flags.sh "$BUILD_PRESET" "$IMAGE_TAG")"
for attempt in 1 2 3; do
- if supersetbot docker \
+ # Supersetbot adds --cache-to whenever DOCKERHUB_TOKEN is present.
+ # Docker auth was already configured above; hide the token from
+ # Supersetbot so release branches and PRs cannot write registry
+ # cache while using --load validation builds.
+ if DOCKERHUB_TOKEN= supersetbot docker \
$PUSH_OR_LOAD \
--preset "$BUILD_PRESET" \
- --context "$EVENT" \
- --context-ref "$RELEASE" $FORCE_LATEST \
+ --context "$BUILD_CONTEXT" \
+ --context-ref "$BUILD_CONTEXT_REF" \
--extra-flags "$EXTRA_FLAGS" \
$PLATFORM_ARG; then
break
@@ -205,7 +431,9 @@ jobs:
# in the context of push (using multi-platform build), we need to pull the image locally
- name: Docker pull
- if: github.event_name == 'push'
+ if: >-
+ steps.publish_guard.outputs.publish == 'true' &&
+ github.event_name == 'push' && github.ref == 'refs/heads/master'
run: |
for i in 1 2 3; do
docker pull $IMAGE_TAG && break
@@ -213,6 +441,7 @@ jobs:
done
- name: Print docker stats
+ if: steps.publish_guard.outputs.publish == 'true'
run: |
echo "SHA: ${{ github.sha }}"
echo "IMAGE: $IMAGE_TAG"
@@ -227,6 +456,7 @@ jobs:
# chain, and is already on the ASF Infra GitHub Actions allowlist.
- name: Scan built image for vulnerabilities
if: >-
+ steps.publish_guard.outputs.publish == 'true' &&
github.event_name == 'push' && github.ref == 'refs/heads/master' &&
contains(fromJson('["superset", "lean"]'), matrix.build_preset)
id: grype-scan
@@ -247,6 +477,7 @@ jobs:
- name: Upload vulnerability scan results to GitHub Security tab
if: >-
+ steps.publish_guard.outputs.publish == 'true' &&
github.event_name == 'push' && github.ref == 'refs/heads/master' &&
contains(fromJson('["superset", "lean"]'), matrix.build_preset) &&
steps.grype-scan.outputs.sarif != ''
@@ -262,7 +493,9 @@ jobs:
category: "docker-${{ matrix.build_preset }}"
- name: WebSocket server smoke test
- if: contains(fromJson('["superset", "lean", "dev"]'), matrix.build_preset)
+ if: >-
+ steps.publish_guard.outputs.publish == 'true' &&
+ contains(fromJson('["superset", "lean", "dev"]'), matrix.build_preset)
shell: bash
run: |
# The realtime WebSocket server is bundled in the official image and
@@ -285,7 +518,9 @@ jobs:
[ "$ok" = "1" ] || { echo "::error::websocket /health did not come up"; exit 1; }
- name: docker-compose sanity check
- if: matrix.build_preset == 'dev'
+ if: >-
+ steps.publish_guard.outputs.publish == 'true' &&
+ matrix.build_preset == 'dev'
shell: bash
env:
BUILD_PRESET: ${{ matrix.build_preset }}
diff --git a/.github/workflows/frontend-bundle-size-nightly.yml b/.github/workflows/frontend-bundle-size-nightly.yml
index 6277f7d3a51..30b35d7e5a7 100644
--- a/.github/workflows/frontend-bundle-size-nightly.yml
+++ b/.github/workflows/frontend-bundle-size-nightly.yml
@@ -40,7 +40,7 @@ jobs:
run: |
docker buildx build \
-t $TAG \
- --cache-from=type=registry,ref=apache/superset-cache:3.11-slim-trixie \
+ --cache-from=type=registry,ref=apache/superset-cache:3.11.14-slim-trixie \
--target superset-node-ci \
.
diff --git a/.github/workflows/scheduled-docker-image-refresh.yml b/.github/workflows/scheduled-docker-image-refresh.yml
deleted file mode 100644
index bb82dfd6605..00000000000
--- a/.github/workflows/scheduled-docker-image-refresh.yml
+++ /dev/null
@@ -1,195 +0,0 @@
-name: Scheduled Docker image refresh
-
-# Re-runs the Docker image build against the latest published release on a
-# weekly cadence. The code being built doesn't change — but the base image
-# layers (python:*-slim-trixie and its OS packages) DO get upstream
-# security patches between Superset releases, and those patches don't
-# reach our published images unless we rebuild.
-#
-# Without this workflow, `apache/superset:<latest>` lags behind upstream
-# Debian/Python base patches by whatever interval falls between Superset
-# releases (typically 3–6 weeks). With it, the lag drops to at most one
-# week regardless of release cadence.
-#
-# This is a security-hygiene cron, not a release. It overwrites the
-# existing tags for the most recent release (e.g. `apache/superset:5.0.0`
-# and `apache/superset:latest`) with bit-for-bit-equivalent contents
-# layered on a refreshed base. Image digests change; everything users
-# actually pin against (image content, code, deps) does not.
-
-on:
- schedule:
- # Mondays at 06:00 UTC — gives the weekend for upstream patches to
- # settle and surfaces failures at the start of the work week so a
- # human can react.
- - cron: "0 6 * * 1"
-
- # Manual trigger so operators can force a refresh on demand (e.g.
- # immediately after a high-severity base-image CVE drops).
- workflow_dispatch: {}
-
-permissions:
- contents: read
-
-# Serialize with itself and with the release publisher (tag-release.yml) —
-# both push to the same Docker Hub tags, so a race could end with stale
-# layers winning. Both workflows must declare this group for the lock to work.
-concurrency:
- group: docker-publish-latest-release
- cancel-in-progress: false
-
-jobs:
- config:
- runs-on: ubuntu-26.04
- outputs:
- has-secrets: ${{ steps.check.outputs.has-secrets }}
- latest-release: ${{ steps.latest.outputs.tag }}
- force-latest: ${{ steps.latest.outputs.force-latest }}
- steps:
- - name: Check for Docker Hub secrets
- id: check
- shell: bash
- run: |
- if [ -n "${DOCKERHUB_USER}" ]; then
- echo "has-secrets=1" >> "$GITHUB_OUTPUT"
- fi
- env:
- DOCKERHUB_USER: ${{ (secrets.DOCKERHUB_USER != '' && secrets.DOCKERHUB_TOKEN != '') || '' }}
-
- - name: Look up latest published release
- id: latest
- shell: bash
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- REPOSITORY: ${{ github.repository }}
- run: |
- # `releases/latest` returns the latest non-prerelease, non-draft
- # release — which is exactly what `apache/superset:latest`
- # should reflect.
- TAG=$(gh api "repos/${REPOSITORY}/releases/latest" --jq .tag_name)
- if [ -z "$TAG" ] || [ "$TAG" = "null" ]; then
- echo "::error::Could not determine latest release tag"
- exit 1
- fi
- echo "Latest release: $TAG"
- echo "tag=$TAG" >> "$GITHUB_OUTPUT"
-
- # Only move `:latest` when the release flagged "latest" is also the
- # highest semver release. This guards against a mis-click leaving an
- # older maintenance release (e.g. a 5.x patch shipped after 6.0 GA)
- # marked latest, which would otherwise roll `:latest` back a major
- # version on the next cron run. If it isn't the newest, we still
- # refresh that release's own version tag but leave `:latest` alone.
- HIGHEST=$(gh api --paginate "repos/${REPOSITORY}/releases" \
- --jq '.[] | select(.draft|not) | select(.prerelease|not) | .tag_name' \
- | sed 's/^v//' | sort -V | tail -n1)
- if [ "${TAG#v}" = "$HIGHEST" ]; then
- echo "force-latest=1" >> "$GITHUB_OUTPUT"
- else
- echo "::warning::Latest-flagged release $TAG is not the highest semver ($HIGHEST); refreshing its version tag but leaving :latest untouched"
- fi
-
- docker-rebuild:
- needs: config
- if: needs.config.outputs.has-secrets == '1'
- name: docker-rebuild
- runs-on: ubuntu-26.04
- strategy:
- # Mirror the same matrix the release publisher uses so every variant
- # operators consume from Docker Hub gets the refreshed base.
- matrix:
- build_preset: ["superset", "dev", "lean", "websocket", "dockerize", "py311", "py312"]
- fail-fast: false
- steps:
- - name: "Checkout release tag: ${{ needs.config.outputs.latest-release }}"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- with:
- ref: ${{ needs.config.outputs.latest-release }}
- fetch-depth: 0
- persist-credentials: false
-
- # Keep workflow tooling on the triggering revision. Release tags can
- # contain action pins that no longer satisfy the repository allowlist.
- - name: Checkout workflow actions
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- with:
- ref: ${{ github.sha }}
- path: workflow-source
- persist-credentials: false
- sparse-checkout: .github/actions
-
- # This action lives under a second, sparse checkout of this same repo
- # into workflow-source/ (see above), not at the repository root, so
- # the $/ self-repository syntax (which resolves paths against the
- # repository itself, not the local workspace) can't reach it. The
- # workspace-relative ./ form is required here.
- - name: Setup Docker Environment
- uses: ./workflow-source/.github/actions/setup-docker # zizmor: ignore[self-repository] - path is a local sparse checkout under workflow-source/, not the repo root; ./ is required here
- with:
- dockerhub-user: ${{ secrets.DOCKERHUB_USER }}
- dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
- install-docker-compose: "false"
- build: "true"
-
- - name: Use Node.js 20
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
- with:
- node-version: 20
-
- # Same reasoning as the Setup Docker Environment step above: this
- # path is under the local workflow-source/ sparse checkout, not the
- # repository root, so $/ can't resolve it.
- - name: Setup supersetbot
- uses: ./workflow-source/.github/actions/setup-supersetbot/ # zizmor: ignore[self-repository] - path is a local sparse checkout under workflow-source/, not the repo root; ./ is required here
-
- - name: Rebuild and push
- env:
- DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- BUILD_PRESET: ${{ matrix.build_preset }}
- LATEST_RELEASE: ${{ needs.config.outputs.latest-release }}
- FORCE_LATEST_FLAG: ${{ needs.config.outputs.force-latest == '1' && '--force-latest' || '' }}
- run: |
- # Reuses the same supersetbot invocation as the release
- # publisher (`tag-release.yml`), so the resulting tags are
- # identical to what a manual release dispatch would produce —
- # just with a freshly-pulled base image layer underneath.
- # `--force-latest` is only passed when the config job confirmed the
- # fetched release is the newest one (see FORCE_LATEST_FLAG above).
- supersetbot docker \
- --push \
- --preset "$BUILD_PRESET" \
- --context release \
- --context-ref "$LATEST_RELEASE" \
- $FORCE_LATEST_FLAG \
- --platform "linux/arm64" \
- --platform "linux/amd64"
-
- # The whole point of this cron is catching base-image CVEs, so a silent
- # failure is the expensive case — a red X in the Actions tab nobody is
- # watching on a Monday. File a tracked issue when any rebuild leg fails so
- # a missed security refresh surfaces instead of sitting unnoticed.
- notify-on-failure:
- needs: [config, docker-rebuild]
- if: failure() && needs.config.outputs.has-secrets == '1'
- runs-on: ubuntu-26.04
- permissions:
- contents: read
- issues: write
- steps:
- - name: Open a tracking issue
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- REPOSITORY: ${{ github.repository }}
- LATEST_RELEASE: ${{ needs.config.outputs.latest-release }}
- RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- run: |
- gh issue create \
- --repo "$REPOSITORY" \
- --title "Scheduled Docker image refresh failed for ${LATEST_RELEASE}" \
- --label "infra:container" \
- --label "#bug" \
- --body "The weekly Docker base-image refresh failed for release \`${LATEST_RELEASE}\`. Published images may be missing upstream base-layer security patches until this is resolved.
-
- Failed run: ${RUN_URL}"
diff --git a/.github/workflows/superset-frontend.yml b/.github/workflows/superset-frontend.yml
index 9b7896c1176..c6456395201 100644
--- a/.github/workflows/superset-frontend.yml
+++ b/.github/workflows/superset-frontend.yml
@@ -57,7 +57,7 @@ jobs:
git show -s --format=raw HEAD
docker buildx build \
-t $TAG \
- --cache-from=type=registry,ref=apache/superset-cache:3.11-slim-trixie \
+ --cache-from=type=registry,ref=apache/superset-cache:3.11.14-slim-trixie \
--target superset-node-ci \
.
diff --git a/.github/workflows/tag-release.yml b/.github/workflows/tag-release.yml
index 72632e76b12..a8e6849b52c 100644
--- a/.github/workflows/tag-release.yml
+++ b/.github/workflows/tag-release.yml
@@ -24,8 +24,8 @@ on:
permissions:
contents: read
-# Serialize with the scheduled Docker image refresh — both workflows push
-# to the same Docker Hub tags and must not race on apache/superset:latest.
+# Serialize release image publishes so concurrent manual runs cannot race on
+# apache/superset:latest.
concurrency:
group: docker-publish-latest-release
cancel-in-progress: false
@@ -55,8 +55,7 @@ jobs:
contents: write
strategy:
matrix:
- build_preset:
- ["superset", "dev", "lean", "websocket", "dockerize", "py311", "py312"]
+ build_preset: ["superset", "dev", "lean", "py311", "py312"]
fail-fast: false
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -64,6 +63,11 @@ jobs:
with:
persist-credentials: false
fetch-depth: 0
+ ref: master
+
+ - name: Preserve Docker flags helper
+ shell: bash
+ run: cp scripts/docker-build-extra-flags.sh "$RUNNER_TEMP/docker-build-extra-flags.sh"
- name: Setup Docker Environment
uses: $/.github/actions/setup-docker
@@ -88,6 +92,7 @@ jobs:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ BUILD_PRESET: ${{ matrix.build_preset }}
INPUT_RELEASE: ${{ github.event.inputs.release }}
INPUT_FORCE_LATEST: ${{ github.event.inputs.force-latest }}
INPUT_GIT_REF: ${{ github.event.inputs.git-ref }}
@@ -104,13 +109,18 @@ jobs:
fi
git checkout "${INPUT_GIT_REF}"
EVENT="release"
+ else
+ git checkout "${GITHUB_EVENT_RELEASE_TAG_NAME}"
fi
+ # Preserve the checked-out release Dockerfile's browser default.
+ EXTRA_FLAGS="$(bash "$RUNNER_TEMP/docker-build-extra-flags.sh" "$BUILD_PRESET" "" release)"
- supersetbot docker \
+ DOCKERHUB_TOKEN= supersetbot docker \
--push \
- --preset ${{ matrix.build_preset }} \
+ --preset "$BUILD_PRESET" \
--context "$EVENT" \
--context-ref "$RELEASE" $FORCE_LATEST \
+ --extra-flags "$EXTRA_FLAGS" \
--platform "linux/arm64" \
--platform "linux/amd64"
diff --git a/UPDATING.md b/UPDATING.md
index 2682a8b1be7..4eed9094bd1 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -128,6 +128,22 @@ but under `-lean` tags: `latest-lean`, `master-lean`, `5.0.0-lean`, `<sha>-lean`
drivers most installations need are now present out of the box.
- The `-dev` images (`latest-dev`, `master-dev`, …) are unchanged.
+### Docker image publishing now excludes standalone `websocket` and `dockerize` images
+
+The Apache Superset Docker Hub repository no longer publishes standalone
+`apache/superset:*websocket` or `apache/superset:*dockerize` image tags. The
+realtime WebSocket server is bundled in the `superset`, `lean`, and `dev` images
+and can be launched with `/app/docker/entrypoints/run-websocket.sh`. Helm init
+containers use the main Superset image for dependency checks.
+
+After this policy is cherry-picked into each active release branch, its pushes
+will validate the Docker build locally instead of publishing Docker Hub images
+or cache layers. Until then, those branches retain their previous publishing
+behavior. Official release tags (`X.Y.Z`, `latest`, and
+their preset variants) are published only by the release workflow after release
+manager sign-off. The scheduled release-image refresh workflow was removed, so
+official release tag digests are not overwritten outside release publishing.
+
### Report capture readiness is rechecked immediately before screenshots
Scheduled report and alert captures require chart readiness to remain stable
diff --git a/docker-compose-light.yml b/docker-compose-light.yml
index 61077f9a87a..41a3499c28b 100644
--- a/docker-compose-light.yml
+++ b/docker-compose-light.yml
@@ -71,7 +71,7 @@ x-common-build: &common-build
context: .
target: ${SUPERSET_BUILD_TARGET:-dev} # dev (default, live-reload); also `superset` (batteries-included) or `lean` (minimal)
cache_from:
- - apache/superset-cache:3.11-slim-trixie
+ - apache/superset-cache:3.11.14-slim-trixie
args:
DEV_MODE: "true"
INCLUDE_CHROMIUM: ${INCLUDE_CHROMIUM:-false}
diff --git a/docker-compose-non-dev.yml b/docker-compose-non-dev.yml
index 8032c44b231..f457e798a14 100644
--- a/docker-compose-non-dev.yml
+++ b/docker-compose-non-dev.yml
@@ -36,7 +36,7 @@ x-common-build: &common-build
# and `superset` bundles the metadata/analytics drivers it needs.
target: superset
cache_from:
- - apache/superset-cache:3.11-slim-trixie
+ - apache/superset-cache:3.11.14-slim-trixie
services:
redis:
diff --git a/docker-compose.yml b/docker-compose.yml
index ac9ddd77ed4..381a1cefca4 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -39,7 +39,7 @@ x-common-build: &common-build
context: .
target: ${SUPERSET_BUILD_TARGET:-dev} # dev (default, live-reload); also `superset` (batteries-included) or `lean` (minimal)
cache_from:
- - apache/superset-cache:3.11-slim-trixie
+ - apache/superset-cache:3.11.14-slim-trixie
args:
DEV_MODE: "true"
INCLUDE_CHROMIUM: ${INCLUDE_CHROMIUM:-false}
diff --git a/docs/admin_docs/installation/docker-builds.mdx b/docs/admin_docs/installation/docker-builds.mdx
index 25d41018283..13f62b43a5b 100644
--- a/docs/admin_docs/installation/docker-builds.mdx
+++ b/docs/admin_docs/installation/docker-builds.mdx
@@ -11,16 +11,21 @@ The Apache Superset community extensively uses Docker for development, release,
and productionizing Superset. This page details our Docker builds and tag naming
schemes to help users navigate our offerings.
-Images are built and pushed to the [Superset Docker Hub repository](https://hub.docker.com/r/apache/superset) using GitHub Actions.
+Images are built, and some of them are pushed to the [Superset Docker Hub repository](https://hub.docker.com/r/apache/superset), using GitHub Actions.
Different sets of images are built and/or published at different times:
-- **Published releases** (`release`): published using
- tags like `5.0.0` and the `latest` tag.
-- **Pull request iterations** (`pull_request`): for each pull request, while
- we actively build the docker to validate the build, we do
- not publish those images for security reasons, we simply `docker build --load`
-- **Merges to the main branch** (`push`): resulting in new SHAs, with tags
- prefixed with `master` for the latest `master` version.
+- **Published releases** (`release`): published by the release workflow after
+ release manager sign-off, using tags like `5.0.0` and the `latest` tag.
+- **Pull request validation** (`pull_request`): pull requests build the
+ `superset`, `dev`, and `lean` presets to validate the Dockerfile. These
+ validation images are loaded into the CI runner and are not published.
+- **Showtime pull request environments**: maintainer-triggered ephemeral
+ environments publish images tagged like `pr-5252-a1b2c3d-ci`.
+- **Merges to the `master` branch** (`push`): published using commit-addressable
+ SHA tags and the `master` tag family.
+- **Merges to release branches** (`push`): built locally for validation only.
+ Official release images are published by the release manager after a release
+ vote completes and the release is signed off.
## Build presets
@@ -28,7 +33,7 @@ We have a set of build "presets" that each represent a combination of
parameters for the build, mostly pointing to either different target layer
for the build, and/or base image.
-Here are the build presets that are exposed through the `supersetbot docker` utility:
+Here are the build presets that are published to Docker Hub:
- `superset`: The default Docker image, including both frontend and backend. Tags
without a build_preset suffix are `superset` builds (ie: `latest`, `5.0.0`, `4.1.2`, ...).
@@ -46,14 +51,19 @@ Here are the build presets that are exposed through the `supersetbot docker` uti
- `dev`: For development, with a headless browser, dev-related utilities and root access. This
includes some commonly used database drivers like `mysqlclient`, `psycopg2-binary` and
some other used for development/CI
-- `py311`, e.g., Py311: Similar to lean but with a different Python version (in this example, 3.11).
-- `ci`: For certain CI workloads.
-- `websocket`: For Superset clusters supporting advanced features.
-- `dockerize`: Used by Helm in initContainers to wait for database dependencies to be available.
+- `py311` and `py312`: Similar to lean but with a different Python version.
+
+Standalone `websocket` and `dockerize` images are not published. The realtime
+WebSocket server is bundled in the `superset`, `lean`, and `dev` images and can
+be started with `/app/docker/entrypoints/run-websocket.sh`. Helm init containers
+use the main Superset image for dependency checks.
The `ci` and `showtime` Docker targets share an entrypoint that runs container
initialization before starting the server and stops if initialization exits with
-a nonzero status. `SERVER_THREADS_AMOUNT` defaults to `8` when unset or empty, while an
+a nonzero status. Neither is part of the published preset matrix above: `ci`
+images stay on the CI runner, while `showtime` images are published by Showtime
+under the `pr-<PR>-<sha>-ci` tags described below.
+`SERVER_THREADS_AMOUNT` defaults to `8` when unset or empty, while an
explicit value is preserved. The server then replaces the entrypoint process so
it receives signals directly and its exit status becomes the container status.
@@ -69,10 +79,9 @@ it receives signals directly and its exit status becomes the container status.
build preset
- `master-lean`: Similar to `master` but the minimal variant with no database drivers.
- `master-dev`: Similar to `master` but includes a headless browser and root access.
-- `pr-5252`: The latest commit in PR 5252.
+- `pr-5252-a1b2c3d-ci`: A Showtime image for a specific pull request commit.
- `30948dc401b40982cb7c0dbf6ebbe443b2748c1b-dev`: A build for
- this specific SHA, which could be from a `master` merge, or release.
-- `websocket-latest`: The WebSocket image for use in a Superset cluster.
+ this specific SHA, which could be from a `master` merge or official release build.
For insights or modifications to the build matrix and tagging conventions,
check the [supersetbot docker](https://github.com/apache-superset/supersetbot)
@@ -94,10 +103,9 @@ have to be enabled in your config file.
You would build the image with `docker build -t mysuperset:latest .` or `docker build -t ourcompanysuperset:5.0.0 .`
```Dockerfile
-# Start from the minimal lean image (no drivers) and add exactly what you need,
-# or start from apache/superset:master to build on top of the batteries-included
-# default (which already has psycopg2-binary, mysqlclient, MCP and Playwright).
-FROM apache/superset:master-lean
+# Start from the minimal lean image (no drivers) and add exactly what you need.
+# Replace this with the release tag you deploy.
+FROM apache/superset:5.0.0-lean
USER root
@@ -215,16 +223,14 @@ recommend deriving from the `-lean` image and adding exactly the database suppor
## On supporting different platforms (namely arm64 AND amd64)
-Currently all automated builds are multi-platform, supporting both `linux/arm64`
-and `linux/amd64`. This enables higher level constructs like `helm` and
-`docker compose` to point to these images and effectively be multi-platform
-as well.
+Published release images and `master` branch push builds are multi-platform,
+supporting both `linux/arm64` and `linux/amd64`. This enables higher level
+constructs like `helm` and `docker compose` to point to these images and
+effectively be multi-platform as well.
-Pull requests and master builds
-are one-image-per-platform so that they can be parallelized and the
-build matrix for those is more sparse as we don't need to build every
-build preset on every platform, and generally can be more selective here.
-For those builds, we suffix tags with `-arm` where it applies.
+Pull request and release branch validation builds use `linux/amd64` and load the
+image into the CI runner. Showtime pull request environment images also target
+`linux/amd64`.
### Working with Apple silicon
diff --git a/docs/admin_docs/installation/docker-compose.mdx b/docs/admin_docs/installation/docker-compose.mdx
index c23ac71b08b..fe4d80af8f5 100644
--- a/docs/admin_docs/installation/docker-compose.mdx
+++ b/docs/admin_docs/installation/docker-compose.mdx
@@ -152,7 +152,7 @@ git checkout $TAG
docker compose -f docker-compose-image-tag.yml up
```
-Here various release tags, github SHA, and latest `master` can be referenced by the TAG env var.
+Here various release tags, Git SHA tags, and `master` tags can be referenced by the TAG env var.
Refer to the docker-related documentation to learn more about existing tags you can point to
from Docker Hub.
diff --git a/scripts/change_detector.py b/scripts/change_detector.py
index 1cca96af302..784ba7fe2d9 100755
--- a/scripts/change_detector.py
+++ b/scripts/change_detector.py
@@ -43,7 +43,6 @@ PATTERNS = {
"python": [
r"^\.github/workflows/.*python",
r"^\.github/workflows/frontend-bundle-size-nightly\.yml$",
- r"^\.github/workflows/scheduled-docker-image-refresh\.yml$",
r"^docker-compose-image-tag\.yml$",
r"^tests/",
r"^superset/",
diff --git a/scripts/docker-build-extra-flags.sh b/scripts/docker-build-extra-flags.sh
index cb46b3cd34a..6b5bd68f46f 100755
--- a/scripts/docker-build-extra-flags.sh
+++ b/scripts/docker-build-extra-flags.sh
@@ -31,18 +31,53 @@
# appending PY_VER here would override supersetbot's pin and silently make
# "py311"/"py312" build the exact same image as "lean". Every other preset
# gets the override so its build lands on the Dockerfile's own supported
-# Python version.
+# Python version. Those presets also point buildx at a matching cache tag so
+# local and CI builds can pull cache layers for the same base image they build.
#
-# Usage: docker-build-extra-flags.sh <build_preset> <image_tag>
+# Usage: docker-build-extra-flags.sh <build_preset> [image_tag] [release]
set -euo pipefail
-BUILD_PRESET="${1:?usage: docker-build-extra-flags.sh <build_preset> <image_tag>}"
-IMAGE_TAG="${2:?usage: docker-build-extra-flags.sh <build_preset> <image_tag>}"
+BUILD_PRESET="${1:?usage: docker-build-extra-flags.sh <build_preset> [image_tag]}"
+IMAGE_TAG="${2:-}"
+BUILD_MODE="${3:-ci}"
+DEFAULT_PY_VER="$(sed -n 's/^ARG PY_VER=//p' Dockerfile | head -n 1)"
+if [ -z "$DEFAULT_PY_VER" ]; then
+ echo "Could not determine the default PY_VER from Dockerfile" >&2
+ exit 1
+fi
+
+EXTRA_FLAGS=""
+if [ "$BUILD_MODE" = "ci" ]; then
+ EXTRA_FLAGS="--build-arg INCLUDE_CHROMIUM=false"
+elif [ "$BUILD_MODE" != "release" ]; then
+ echo "Unknown Docker build mode: $BUILD_MODE" >&2
+ exit 1
+fi
+if [ -n "$IMAGE_TAG" ]; then
+ EXTRA_FLAGS="${EXTRA_FLAGS:+$EXTRA_FLAGS }--tag $IMAGE_TAG"
+fi
+case "$BUILD_PRESET" in
+ py311)
+ CACHE_REF="apache/superset-cache:3.11-slim-bookworm"
+ ;;
+ py312)
+ CACHE_REF="apache/superset-cache:3.12-slim-bookworm"
+ ;;
+ *)
+ CACHE_REF="apache/superset-cache:${DEFAULT_PY_VER}"
+ # Keep the superset cache ref for Compose consumers; isolate other
+ # matrix targets so concurrent exports cannot overwrite its layers.
+ if [ "$BUILD_PRESET" != "superset" ]; then
+ CACHE_REF="${CACHE_REF}-${BUILD_PRESET}"
+ fi
+ EXTRA_FLAGS="--build-arg PY_VER=$DEFAULT_PY_VER${EXTRA_FLAGS:+ $EXTRA_FLAGS}"
+ ;;
+esac
-EXTRA_FLAGS="--build-arg INCLUDE_CHROMIUM=false --tag $IMAGE_TAG"
-if [ "$BUILD_PRESET" != "py311" ] && [ "$BUILD_PRESET" != "py312" ]; then
- EXTRA_FLAGS="--build-arg PY_VER=3.11.14-slim-trixie $EXTRA_FLAGS"
+EXTRA_FLAGS="--cache-from=type=registry,ref=$CACHE_REF $EXTRA_FLAGS"
+if [ "${PUBLISH_DOCKER_CACHE:-}" = "true" ] && [ -n "${DOCKERHUB_TOKEN:-}" ]; then
+ EXTRA_FLAGS="--cache-to=type=registry,mode=max,ref=$CACHE_REF $EXTRA_FLAGS"
fi
echo "$EXTRA_FLAGS"
diff --git a/scripts/docker-build-plan.sh b/scripts/docker-build-plan.sh
new file mode 100644
index 00000000000..62e15edd00c
--- /dev/null
+++ b/scripts/docker-build-plan.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+EVENT_NAME="${1:-${GITHUB_EVENT_NAME:-}}"
+REF="${2:-${GITHUB_REF:-}}"
+REF_NAME="${3:-${GITHUB_REF_NAME:-}}"
+
+BUILD_CONTEXT="$EVENT_NAME"
+BUILD_CONTEXT_REF=""
+PUBLISH_DOCKER_CACHE=""
+
+if [ "$EVENT_NAME" = "push" ] && [ "$REF" = "refs/heads/master" ]; then
+ PLATFORM_ARG="--platform linux/arm64 --platform linux/amd64"
+ PUSH_OR_LOAD="--push"
+ BUILD_CONTEXT_REF="${REF_NAME:-master}"
+ PUBLISH_DOCKER_CACHE="true"
+elif [ "$EVENT_NAME" = "push" ] || [ "$EVENT_NAME" = "pull_request" ]; then
+ PLATFORM_ARG="--platform linux/amd64"
+ PUSH_OR_LOAD="--load"
+else
+ echo "Unsupported GitHub event for docker build: $EVENT_NAME" >&2
+ exit 1
+fi
+
+emit() {
+ printf '%s=%q\n' "$1" "$2"
+}
+
+emit BUILD_CONTEXT "$BUILD_CONTEXT"
+emit BUILD_CONTEXT_REF "$BUILD_CONTEXT_REF"
+emit PLATFORM_ARG "$PLATFORM_ARG"
+emit PUSH_OR_LOAD "$PUSH_OR_LOAD"
+emit PUBLISH_DOCKER_CACHE "$PUBLISH_DOCKER_CACHE"
diff --git a/scripts/docker-publish-guard.sh b/scripts/docker-publish-guard.sh
new file mode 100644
index 00000000000..0571818c6d4
--- /dev/null
+++ b/scripts/docker-publish-guard.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Decides whether a docker-build job may publish the mutable `master*` tags.
+#
+# Concurrency serializes the master matrix per preset, but it does not order
+# the runs that enter it: if push A's change-detection job is slow and push B's
+# is fast, B acquires the lock and publishes, then A acquires it and
+# republishes `master`, `master-dev`, ... from the older commit. Comparing the
+# building SHA against the branch tip at that point rejects A.
+#
+# Only master publishing is gated. Release branches and pull requests build
+# with --load and move no mutable tag, so they always proceed.
+#
+# Usage: docker-publish-guard.sh <event_name> <ref> <sha> <tip_sha>
+# Emits: PUBLISH=true|false
+
+set -euo pipefail
+
+EVENT_NAME="${1:?usage: docker-publish-guard.sh <event_name> <ref> <sha> <tip_sha>}"
+REF="${2:-}"
+SHA="${3:-}"
+TIP_SHA="${4:-}"
+
+PUBLISH="true"
+if [ "$EVENT_NAME" = "push" ] && [ "$REF" = "refs/heads/master" ]; then
+ if [ -z "$SHA" ] || [ -z "$TIP_SHA" ]; then
+ # An unresolvable tip (network failure, empty ref) must not silently
+ # degrade into publishing from a possibly stale commit.
+ echo "Could not compare the building SHA against the master tip" >&2
+ exit 1
+ fi
+ if [ "$SHA" != "$TIP_SHA" ]; then
+ PUBLISH="false"
+ fi
+fi
+
+printf 'PUBLISH=%q\n' "$PUBLISH"
diff --git a/tests/unit_tests/scheduled_docker_image_refresh_test.py b/tests/unit_tests/scheduled_docker_image_refresh_test.py
deleted file mode 100644
index fd57eff4707..00000000000
--- a/tests/unit_tests/scheduled_docker_image_refresh_test.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from pathlib import Path
-from typing import Any
-
-import yaml
-
-from scripts import change_detector
-
-WORKFLOW_PATH = (
- Path(__file__).resolve().parents[2]
- / ".github/workflows/scheduled-docker-image-refresh.yml"
-)
-
-
-def load_workflow() -> dict[str, Any]:
- return yaml.safe_load(WORKFLOW_PATH.read_text())
-
-
-def test_scheduled_refresh_uses_current_workflow_actions() -> None:
- workflow = load_workflow()
- steps = {step["name"]: step for step in workflow["jobs"]["docker-rebuild"]["steps"]}
-
- action_checkout = steps["Checkout workflow actions"]
- assert action_checkout["with"]["ref"] == "${{ github.sha }}"
- assert action_checkout["with"]["path"] == "workflow-source"
- assert (
- steps["Setup Docker Environment"]["uses"]
- == "./workflow-source/.github/actions/setup-docker"
- )
- assert (
- steps["Setup supersetbot"]["uses"]
- == "./workflow-source/.github/actions/setup-supersetbot/"
- )
-
-
-def test_scheduled_refresh_notifier_uses_existing_labels() -> None:
- workflow = load_workflow()
- notify_step = workflow["jobs"]["notify-on-failure"]["steps"][0]
-
- assert '--label "infra:container"' in notify_step["run"]
- assert '--label "#bug"' in notify_step["run"]
- assert '--label "bug"' not in notify_step["run"]
-
-
-def test_scheduled_refresh_changes_trigger_python_tests() -> None:
- assert change_detector.detect_changes(
- [".github/workflows/scheduled-docker-image-refresh.yml"],
- change_detector.PATTERNS["python"],
- )
diff --git a/tests/unit_tests/scripts/docker_cache_ref_test.py b/tests/unit_tests/scripts/docker_cache_ref_test.py
new file mode 100644
index 00000000000..573321d1f52
--- /dev/null
+++ b/tests/unit_tests/scripts/docker_cache_ref_test.py
@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Binds the hard-coded ``apache/superset-cache`` consumers to the Dockerfile.
+
+``scripts/docker-build-extra-flags.sh`` derives the cache tag it *exports* from
+the Dockerfile's ``ARG PY_VER`` default, but Compose files and the frontend
+workflows spell the tag they *import* out in full. Without this check a routine
+``PY_VER`` bump moves the exporter while every consumer keeps pulling a tag
+nobody writes any more, silently turning warm builds into cold ones.
+"""
+
+import re
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).parents[3]
+
+# Files that hard-code a cache tag and must track the Dockerfile default.
+# scripts/docker-build-extra-flags.sh is deliberately absent: its py311/py312
+# refs pin deliberately different base images.
+CACHE_REF_CONSUMERS = (
+ "docker-compose.yml",
+ "docker-compose-non-dev.yml",
+ "docker-compose-light.yml",
+ ".github/workflows/superset-frontend.yml",
+ ".github/workflows/frontend-bundle-size-nightly.yml",
+)
+
+# Suffix `scripts/docker-build-extra-flags.sh` appends to isolate a matrix
+# target from the shared `superset` cache ref. Matched generically rather than
+# against a list of known presets, so adding a preset there cannot leave this
+# check stale; the invariant being enforced is the base Python version, not
+# which presets exist.
+CACHE_REF_PATTERN = re.compile(r"apache/superset-cache:([^\s\"']+)")
+PRESET_SUFFIX_PATTERN = r"(?:-[a-z0-9]+)?"
+
+
+def _dockerfile_py_ver() -> str:
+ """Returns the Dockerfile's default ``ARG PY_VER``, the tag's source of truth.
+
+ Mirrors the ``DEFAULT_PY_VER`` extraction in
+ ``scripts/docker-build-extra-flags.sh``, which is what makes the exported
+ cache tag track the Dockerfile in the first place.
+ """
+ for line in (REPO_ROOT / "Dockerfile").read_text().splitlines():
+ if line.startswith("ARG PY_VER="):
+ return line.removeprefix("ARG PY_VER=").strip()
+ raise AssertionError("Dockerfile no longer declares a default ARG PY_VER")
+
+
+@pytest.mark.parametrize("relative_path", CACHE_REF_CONSUMERS)
+def test_cache_ref_matches_dockerfile_py_ver(relative_path: str) -> None:
+ """Every hard-coded cache tag pulls the base image the Dockerfile declares.
+
+ Fails when a ``PY_VER`` bump moves the tag
+ ``scripts/docker-build-extra-flags.sh`` exports to without updating the
+ files that import it, which would otherwise leave them pulling a tag
+ nothing writes any more.
+ """
+ path = REPO_ROOT / relative_path
+ assert path.exists(), f"{relative_path} moved; update CACHE_REF_CONSUMERS"
+
+ py_ver = _dockerfile_py_ver()
+ allowed = re.compile(rf"^{re.escape(py_ver)}{PRESET_SUFFIX_PATTERN}$")
+
+ refs = CACHE_REF_PATTERN.findall(path.read_text())
+ assert refs, f"{relative_path} no longer references apache/superset-cache"
+
+ for ref in refs:
+ assert allowed.match(ref), (
+ f"{relative_path} pulls apache/superset-cache:{ref}, but "
+ f"scripts/docker-build-extra-flags.sh exports to "
+ f"apache/superset-cache:{py_ver} (from the Dockerfile's "
+ f"ARG PY_VER). Bump the two together."
+ )