You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ma...@apache.org on 2024/01/19 23:34:55 UTC
(superset) 01/06: build: Parallelize the CI image builds
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch parallelize_image_builds
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 82b0b90fa7111719f71ef39bae13ba45e4979d94
Author: sebastianliebscher <li...@protonmail.ch>
AuthorDate: Sun Oct 8 09:04:24 2023 +0200
build: Parallelize the CI image builds
- replaces custom script to set tags with docker/metadata-action GitHub Action
- replaces custom script to sequentially build images with docker/build-push-action GitHub Action
- moves docker-release.yml logic into docker.yml by utilizing docker/metadata-action 'tags: type=pep440,pattern={{version}}'
- removes docker buildx local cache usage as every build runs on its own job hence on different machines (docker buildx registry cache will be a follow-up PR)
---
.github/workflows/docker-release.yml | 42 -------
.github/workflows/docker.yml | 207 ++++++++++++++++++++++++++++++++---
scripts/docker_build_push.sh | 189 --------------------------------
3 files changed, 192 insertions(+), 246 deletions(-)
diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml
deleted file mode 100644
index 7cfba73299..0000000000
--- a/.github/workflows/docker-release.yml
+++ /dev/null
@@ -1,42 +0,0 @@
-name: Docker
-
-on:
- release:
- types: [published]
-jobs:
- config:
- runs-on: "ubuntu-latest"
- outputs:
- has-secrets: ${{ steps.check.outputs.has-secrets }}
- steps:
- - name: "Check for secrets"
- id: check
- shell: bash
- run: |
- if [ -n "${{ (secrets.DOCKERHUB_USER != '' && secrets.DOCKERHUB_TOKEN != '') || '' }}" ]; then
- echo "has-secrets=1" >> "$GITHUB_OUTPUT"
- fi
-
- docker-release:
- needs: config
- if: needs.config.outputs.has-secrets
- name: docker-release
- runs-on: ubuntu-latest
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v3
- with:
- persist-credentials: false
- submodules: recursive
- ref: ${{ github.ref }}
- - name: Set up QEMU
- uses: docker/setup-qemu-action@v1
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v1
- - shell: bash
- env:
- DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
- run: |
- GITHUB_RELEASE_TAG_NAME="${{ github.event.release.tag_name }}"
- ./scripts/docker_build_push.sh "$GITHUB_RELEASE_TAG_NAME"
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 6160d3cc1f..1cfa1232ae 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -1,12 +1,15 @@
name: Docker
on:
+ release:
+ types: [ published ]
push:
branches:
- 'master'
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
+
jobs:
config:
runs-on: "ubuntu-latest"
@@ -25,29 +28,205 @@ jobs:
echo "no secrets!"
fi
- docker-build:
+
+ build-lean-image:
+ name: Build ${{ matrix.image.version }} lean image
needs: config
if: needs.config.outputs.has-secrets
- name: docker-build
runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ image:
+ - {version: "3.9-slim-bookworm", suffix: ""}
+ - {version: "3.10-slim-bookworm", suffix: "-py310"}
steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v3
+ - name: Checkout
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ - name: Docker meta
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ github.repository }}
+ flavor: |
+ latest=false
+ suffix=${{ matrix.image.suffix }}
+ tags: |
+ type=sha,prefix=,format=long
+ type=ref,event=pr
+ type=raw,value=master,enable={{is_default_branch}}
+ type=raw,value=latest,enable={{is_default_branch}}
+ type=pep440,pattern={{version}}
+ labels: |
+ target=lean
+ build_actor=${{ github.actor }}
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+ - name: Login to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - name: Build and push
+ uses: docker/build-push-action@v5
+ with:
+ context: .
+ platforms: linux/amd64
+ push: true
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+ target: lean
+ build-args: |
+ PY_VER=${{ matrix.image.version }}
+
+
+ build-dev-image:
+ name: Build dev image
+ needs: config
+ if: needs.config.outputs.has-secrets
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ - name: Docker meta
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ github.repository }}
+ flavor: |
+ latest=false
+ suffix=dev
+ tags: |
+ type=sha,prefix=,format=long
+ type=ref,event=pr
+ type=raw,value=master,enable={{is_default_branch}}
+ type=raw,value=latest,enable={{is_default_branch}}
+ type=pep440,pattern={{version}}
+ labels: |
+ target=dev
+ build_actor=${{ github.actor }}
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+ - name: Login to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - name: Build and push
+ uses: docker/build-push-action@v5
+ with:
+ context: .
+ platforms: linux/amd64
+ push: true
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+ target: dev
+
+
+ build-websocket-image:
+ name: Build websocket image
+ needs: config
+ if: needs.config.outputs.has-secrets
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
with:
persist-credentials: false
- name: Set up QEMU
- uses: docker/setup-qemu-action@v1
+ uses: docker/setup-qemu-action@v3
+ - name: Docker meta
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ github.repository }}
+ flavor: |
+ latest=false
+ suffix=websocket
+ tags: |
+ type=sha,prefix=,format=long
+ type=ref,event=pr
+ type=raw,value=master,enable={{is_default_branch}}
+ type=raw,value=latest,enable={{is_default_branch}}
+ type=pep440,pattern={{version}}
+ labels: |
+ build_actor=${{ github.actor }}
- name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v1
- - shell: bash
- env:
- DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
- run: |
- ./scripts/docker_build_push.sh
+ uses: docker/setup-buildx-action@v3
+ - name: Login to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - name: Build and push
+ uses: docker/build-push-action@v5
+ with:
+ context: ./superset-websocket
+ platforms: linux/amd64,linux/arm64
+ push: true
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+
+ build-dockerize-image:
+ name: Build Dockerize image
+ needs: config
+ if: needs.config.outputs.has-secrets
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+ - name: Docker meta
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ github.repository }}
+ flavor: |
+ latest=false
+ tags: |
+ type=raw,value=dockerize,enable={{is_default_branch}}
+ labels: |
+ build_actor=${{ github.actor }}
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+ - name: Login to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - name: Build and push
+ uses: docker/build-push-action@v5
+ with:
+ context: .
+ file: dockerize.Dockerfile
+ platforms: linux/amd64,linux/arm64
+ push: true
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+
+
+ # build Dockerfile 'ci' target, save to archive and upload as artifact
+ build-ephemeral-image:
+ name: Build ephemeral env image
+ needs: config
+ if: needs.config.outputs.has-secrets && github.event_name == 'pull_request'
+ runs-on: ubuntu-latest
+ steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v3
+ with:
+ persist-credentials: false
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
- name: Build ephemeral env image
- if: github.event_name == 'pull_request'
run: |
mkdir -p ./build
echo ${{ github.sha }} > ./build/SHA
@@ -61,9 +240,7 @@ jobs:
--label "build_actor=${GITHUB_ACTOR}" \
.
docker save ${{ github.sha }} | gzip > ./build/${{ github.sha }}.tar.gz
-
- name: Upload build artifacts
- if: github.event_name == 'pull_request'
uses: actions/upload-artifact@v3
with:
name: build
diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh
deleted file mode 100755
index 8ae82faaeb..0000000000
--- a/scripts/docker_build_push.sh
+++ /dev/null
@@ -1,189 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-set -eo pipefail
-
-GITHUB_RELEASE_TAG_NAME="$1"
-
-SHA=$(git rev-parse HEAD)
-REPO_NAME="apache/superset"
-
-if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
- REFSPEC=$(echo "${GITHUB_HEAD_REF}" | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
- PR_NUM=$(echo "${GITHUB_REF}" | sed 's:refs/pull/::' | sed 's:/merge::')
- LATEST_TAG="pr-${PR_NUM}"
-elif [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then
- REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/tags/::' | head -c 40)
- LATEST_TAG="${REFSPEC}"
-else
- REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/heads/::' | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
- LATEST_TAG="${REFSPEC}"
-fi
-
-
-if [[ "${REFSPEC}" == "master" ]]; then
- LATEST_TAG="master"
-fi
-
-# get the latest release tag
-if [ -n "${GITHUB_RELEASE_TAG_NAME}" ]; then
- output=$(source ./scripts/tag_latest_release.sh "${GITHUB_RELEASE_TAG_NAME}" --dry-run) || true
- SKIP_TAG=$(echo "${output}" | grep "SKIP_TAG" | cut -d'=' -f2)
- if [[ "${SKIP_TAG}" == "SKIP_TAG::false" ]]; then
- LATEST_TAG="latest"
- fi
-fi
-
-if [[ "${TEST_ENV}" == "true" ]]; then
- # don't run the build in test environment
- echo "LATEST_TAG is ${LATEST_TAG}"
- exit 0
-fi
-
-
-cat<<EOF
- Rolling with tags:
- - ${REPO_NAME}:${SHA}
- - ${REPO_NAME}:${REFSPEC}
- - ${REPO_NAME}:${LATEST_TAG}
-EOF
-
-if [ -z "${DOCKERHUB_TOKEN}" ]; then
- # Skip if secrets aren't populated -- they're only visible for actions running in the repo (not on forks)
- echo "Skipping Docker push"
- # By default load it back
- DOCKER_ARGS="--load"
- ARCHITECTURE_FOR_BUILD="linux/amd64 linux/arm64"
-else
- # Login and push
- docker logout
- docker login --username "${DOCKERHUB_USER}" --password "${DOCKERHUB_TOKEN}"
- DOCKER_ARGS="--push"
- ARCHITECTURE_FOR_BUILD="linux/amd64,linux/arm64"
-fi
-set -x
-
-# for the dev image, it's ok to tag master as latest-dev
-# for production, we only want to tag the latest official release as latest
-if [ "${LATEST_TAG}" = "master" ]; then
- DEV_TAG="${REPO_NAME}:latest-dev"
-else
- DEV_TAG="${REPO_NAME}:${LATEST_TAG}-dev"
-fi
-
-for BUILD_PLATFORM in $ARCHITECTURE_FOR_BUILD; do
-#
-# Build the dev image
-#
-docker buildx build --target dev \
- $DOCKER_ARGS \
- --cache-from=type=registry,ref=apache/superset:master-dev \
- --cache-from=type=local,src=/tmp/superset \
- --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
- -t "${REPO_NAME}:${SHA}-dev" \
- -t "${REPO_NAME}:${REFSPEC}-dev" \
- -t "${DEV_TAG}" \
- --platform ${BUILD_PLATFORM} \
- --label "sha=${SHA}" \
- --label "built_at=$(date)" \
- --label "target=dev" \
- --label "build_actor=${GITHUB_ACTOR}" \
- .
-
-#
-# Build the "lean" image
-#
-docker buildx build --target lean \
- $DOCKER_ARGS \
- --cache-from=type=local,src=/tmp/superset \
- --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
- -t "${REPO_NAME}:${SHA}" \
- -t "${REPO_NAME}:${REFSPEC}" \
- -t "${REPO_NAME}:${LATEST_TAG}" \
- --platform ${BUILD_PLATFORM} \
- --label "sha=${SHA}" \
- --label "built_at=$(date)" \
- --label "target=lean" \
- --label "build_actor=${GITHUB_ACTOR}" \
- .
-
-#
-# Build the "lean310" image
-#
-docker buildx build --target lean \
- $DOCKER_ARGS \
- --cache-from=type=local,src=/tmp/superset \
- --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
- -t "${REPO_NAME}:${SHA}-py310" \
- -t "${REPO_NAME}:${REFSPEC}-py310" \
- -t "${REPO_NAME}:${LATEST_TAG}-py310" \
- --platform ${BUILD_PLATFORM} \
- --build-arg PY_VER="3.10-slim-bookworm"\
- --label "sha=${SHA}" \
- --label "built_at=$(date)" \
- --label "target=lean310" \
- --label "build_actor=${GITHUB_ACTOR}" \
- .
-
-#
-# Build the "lean39" image
-#
-docker buildx build --target lean \
- $DOCKER_ARGS \
- --cache-from=type=local,src=/tmp/superset \
- --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
- -t "${REPO_NAME}:${SHA}-py39" \
- -t "${REPO_NAME}:${REFSPEC}-py39" \
- -t "${REPO_NAME}:${LATEST_TAG}-py39" \
- --platform ${BUILD_PLATFORM} \
- --build-arg PY_VER="3.9-slim-bullseye"\
- --label "sha=${SHA}" \
- --label "built_at=$(date)" \
- --label "target=lean39" \
- --label "build_actor=${GITHUB_ACTOR}" \
- .
-
-#
-# Build the "websocket" image
-#
-docker buildx build \
- $DOCKER_ARGS \
- --cache-from=type=registry,ref=apache/superset:master-websocket \
- -t "${REPO_NAME}:${SHA}-websocket" \
- -t "${REPO_NAME}:${REFSPEC}-websocket" \
- -t "${REPO_NAME}:${LATEST_TAG}-websocket" \
- --platform ${BUILD_PLATFORM} \
- --label "sha=${SHA}" \
- --label "built_at=$(date)" \
- --label "target=websocket" \
- --label "build_actor=${GITHUB_ACTOR}" \
- superset-websocket
-
-#
-# Build the dockerize image
-#
-docker buildx build \
- $DOCKER_ARGS \
- --cache-from=type=registry,ref=apache/superset:dockerize \
- -t "${REPO_NAME}:dockerize" \
- --platform ${BUILD_PLATFORM} \
- --label "sha=${SHA}" \
- --label "built_at=$(date)" \
- --label "build_actor=${GITHUB_ACTOR}" \
- -f dockerize.Dockerfile \
- .
-done