You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by el...@apache.org on 2023/10/18 23:59:38 UTC
[superset] branch master updated: chore: add latest-official docker tag (#25322)
This is an automated email from the ASF dual-hosted git repository.
elizabeth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 26498fc099 chore: add latest-official docker tag (#25322)
26498fc099 is described below
commit 26498fc09921bbfc5a2fbaa286586b07ca5835e3
Author: Elizabeth Thompson <es...@gmail.com>
AuthorDate: Wed Oct 18 16:59:30 2023 -0700
chore: add latest-official docker tag (#25322)
---
.github/workflows/docker-release.yml | 3 +-
.github/workflows/docker.yml | 2 +-
.../workflows => scripts}/docker_build_push.sh | 51 +++++++-
scripts/tag_latest_release.sh | 140 +++++++++++++++------
tests/unit_tests/fixtures/bash_mock.py | 44 +++++++
tests/unit_tests/scripts/docker_build_push_test.py | 44 +++++++
.../unit_tests/scripts/tag_latest_release_test.py | 49 ++++++++
7 files changed, 290 insertions(+), 43 deletions(-)
diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml
index b079206f51..7cfba73299 100644
--- a/.github/workflows/docker-release.yml
+++ b/.github/workflows/docker-release.yml
@@ -38,4 +38,5 @@ jobs:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
run: |
- .github/workflows/docker_build_push.sh
+ GITHUB_RELEASE_TAG_NAME="${{ github.event.release.tag_name }}"
+ ./scripts/docker_build_push.sh "$GITHUB_RELEASE_TAG_NAME"
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 983267ff60..6160d3cc1f 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -44,7 +44,7 @@ jobs:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
run: |
- .github/workflows/docker_build_push.sh
+ ./scripts/docker_build_push.sh
- name: Build ephemeral env image
if: github.event_name == 'pull_request'
diff --git a/.github/workflows/docker_build_push.sh b/scripts/docker_build_push.sh
similarity index 76%
rename from .github/workflows/docker_build_push.sh
rename to scripts/docker_build_push.sh
index 3e90af45f1..80d08c47e3 100755
--- a/.github/workflows/docker_build_push.sh
+++ b/scripts/docker_build_push.sh
@@ -17,6 +17,8 @@
#
set -eo pipefail
+GITHUB_RELEASE_TAG_NAME="$1"
+
SHA=$(git rev-parse HEAD)
REPO_NAME="apache/superset"
@@ -32,10 +34,27 @@ else
LATEST_TAG="${REFSPEC}"
fi
+
if [[ "${REFSPEC}" == "master" ]]; then
- LATEST_TAG="latest"
+ LATEST_TAG="master"
+fi
+
+# get the latest release tag
+if [ -n "${GITHUB_RELEASE_TAG_NAME}" ]; then
+ output=$(source ./scripts/tag_latest_release.sh "${GITHUB_RELEASE_TAG_NAME}" --dry-run) || true
+ SKIP_TAG=$(echo "${output}" | grep "SKIP_TAG" | cut -d'=' -f2)
+ if [[ "${SKIP_TAG}" == "SKIP_TAG::false" ]]; then
+ LATEST_TAG="latest"
+ fi
+fi
+
+if [[ "${TEST_ENV}" == "true" ]]; then
+ # don't run the build in test environment
+ echo "LATEST_TAG is ${LATEST_TAG}"
+ exit 0
fi
+
cat<<EOF
Rolling with tags:
- ${REPO_NAME}:${SHA}
@@ -58,6 +77,14 @@ else
fi
set -x
+# for the dev image, it's ok to tag master as latest-dev
+# for production, we only want to tag the latest official release as latest
+if [ "${LATEST_TAG}" = "master" ]; then
+ DEV_TAG="${REPO_NAME}:latest-dev"
+else
+ DEV_TAG="${REPO_NAME}:${LATEST_TAG}-dev"
+fi
+
#
# Build the dev image
#
@@ -68,7 +95,7 @@ docker buildx build --target dev \
--cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-t "${REPO_NAME}:${SHA}-dev" \
-t "${REPO_NAME}:${REFSPEC}-dev" \
- -t "${REPO_NAME}:${LATEST_TAG}-dev" \
+ -t "${DEV_TAG}" \
--platform linux/amd64 \
--label "sha=${SHA}" \
--label "built_at=$(date)" \
@@ -110,6 +137,26 @@ docker buildx build --target lean \
--label "target=lean310" \
--label "build_actor=${GITHUB_ACTOR}" \
.
+
+#
+# Build the "lean39" image
+#
+docker buildx build --target lean \
+ $DOCKER_ARGS \
+ --cache-from=type=local,src=/tmp/superset \
+ --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
+ -t "${REPO_NAME}:${SHA}-py39" \
+ -t "${REPO_NAME}:${REFSPEC}-py39" \
+ -t "${REPO_NAME}:${LATEST_TAG}-py39" \
+ --platform linux/amd64 \
+ --build-arg PY_VER="3.9-slim-bullseye"\
+ --label "sha=${SHA}" \
+ --label "built_at=$(date)" \
+ --label "target=lean39" \
+ --label "build_actor=${GITHUB_ACTOR}" \
+ .
+
+
for BUILD_PLATFORM in $ARCHITECTURE_FOR_BUILD; do
#
# Build the "websocket" image
diff --git a/scripts/tag_latest_release.sh b/scripts/tag_latest_release.sh
index a5d50eaf9f..f2ee846a63 100755
--- a/scripts/tag_latest_release.sh
+++ b/scripts/tag_latest_release.sh
@@ -17,7 +17,7 @@
#
run_git_tag () {
- if [ "$DRY_RUN" = "false" ] && [ "$SKIP_TAG" = "false" ]
+ if [[ "$DRY_RUN" == "false" ]] && [[ "$SKIP_TAG" == "false" ]]
then
git tag -a -f latest "${GITHUB_TAG_NAME}" -m "latest tag"
echo "${GITHUB_TAG_NAME} has been tagged 'latest'"
@@ -25,7 +25,55 @@ run_git_tag () {
exit 0
}
-echo "::set-output name=SKIP_TAG::false"
+###
+# separating out git commands into functions so they can be mocked in unit tests
+###
+git_show_ref () {
+ if [[ "$TEST_ENV" == "true" ]]
+ then
+ if [[ "$GITHUB_TAG_NAME" == "does_not_exist" ]]
+ # mock return for testing only
+ then
+ echo ""
+ else
+ echo "2817aebd69dc7d199ec45d973a2079f35e5658b6 refs/tags/${GITHUB_TAG_NAME}"
+ fi
+ fi
+ result=$(git show-ref "${GITHUB_TAG_NAME}")
+ echo "${result}"
+}
+
+get_latest_tag_list () {
+ if [[ "$TEST_ENV" == "true" ]]
+ then
+ echo "(tag: 2.1.0, apache/2.1test)"
+ else
+ result=$(git show-ref --tags --dereference latest | awk '{print $2}' | xargs git show --pretty=tformat:%d -s | grep tag:)
+ echo "${result}"
+ fi
+}
+###
+
+split_string () {
+ local version="$1"
+ local delimiter="$2"
+ local components=()
+ local tmp=""
+ for (( i=0; i<${#version}; i++ )); do
+ local char="${version:$i:1}"
+ if [[ "$char" != "$delimiter" ]]; then
+ tmp="$tmp$char"
+ elif [[ -n "$tmp" ]]; then
+ components+=("$tmp")
+ tmp=""
+ fi
+ done
+ if [[ -n "$tmp" ]]; then
+ components+=("$tmp")
+ fi
+ echo "${components[@]}"
+}
+
DRY_RUN=false
# get params passed in with script when it was run
@@ -50,12 +98,14 @@ done
if [ -z "${GITHUB_TAG_NAME}" ]; then
echo "Missing tag parameter, usage: ./scripts/tag_latest_release.sh <GITHUB_TAG_NAME>"
+ echo "::set-output name=SKIP_TAG::true"
exit 1
fi
-if [ -z "$(git show-ref ${GITHUB_TAG_NAME})" ]; then
+if [ -z "$(git_show_ref)" ]; then
echo "The tag ${GITHUB_TAG_NAME} does not exist. Please use a different tag."
- exit 1
+ echo "::set-output name=SKIP_TAG::true"
+ exit 0
fi
# check that this tag only contains a proper semantic version
@@ -63,36 +113,41 @@ if ! [[ ${GITHUB_TAG_NAME} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
then
echo "This tag ${GITHUB_TAG_NAME} is not a valid release version. Not tagging."
echo "::set-output name=SKIP_TAG::true"
- exit 0
+ exit 1
fi
## split the current GITHUB_TAG_NAME into an array at the dot
-IFS=$'.'
-THIS_TAG_NAME=(${GITHUB_TAG_NAME}) || echo 'not found'
+THIS_TAG_NAME=$(split_string "${GITHUB_TAG_NAME}" ".")
# look up the 'latest' tag on git
-LATEST_TAG_LIST=$(git show-ref latest && git show --pretty=tformat:%d -s latest | grep tag:) || echo 'not found'
+LATEST_TAG_LIST=$(get_latest_tag_list) || echo 'not found'
# if 'latest' tag doesn't exist, then set this commit to latest
if [[ -z "$LATEST_TAG_LIST" ]]
then
- # move on to next task
echo "there are no latest tags yet, so I'm going to start by tagging this sha as the latest"
run_git_tag
+ exit 0
fi
-## get all tags that use the same sha as the latest tag. split at comma.
-IFS=$','
-LATEST_TAGS=($LATEST_TAG_LIST)
+# remove parenthesis and tag: from the list of tags
+LATEST_TAGS_STRINGS=$(echo "$LATEST_TAG_LIST" | sed 's/tag: \([^,]*\)/\1/g' | tr -d '()')
-## loop over those tags and only take action on the one that isn't tagged 'latest'
-## that one will have the version number tag
-for (( i=0; i<${#LATEST_TAGS[@]}; i++ ))
+LATEST_TAGS=$(split_string "$LATEST_TAGS_STRINGS" ",")
+TAGS=($(split_string "$LATEST_TAGS" " "))
+
+# Initialize a flag for comparison result
+compare_result=""
+
+# Iterate through the tags of the latest release
+for tag in $TAGS
do
- if [[ ${LATEST_TAGS[$i]} != *"latest"* ]]
- then
+ if [[ $tag == "latest" ]]; then
+ continue
+ else
## extract just the version from this tag
- LATEST_RELEASE_TAG=$(echo "${LATEST_TAGS[$i]}" | sed -E -e 's/tag:|\(|\)|[[:space:]]*//g')
+ LATEST_RELEASE_TAG="$tag"
+ echo "LATEST_RELEASE_TAG: ${LATEST_RELEASE_TAG}"
# check that this only contains a proper semantic version
if ! [[ ${LATEST_RELEASE_TAG} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
@@ -101,28 +156,35 @@ do
continue
fi
echo "The current release with the latest tag is version ${LATEST_RELEASE_TAG}"
-
- ## remove the sha from the latest tag and split into an array- split at the dot
- IFS=$'.'
- LATEST_RELEASE_TAG_SPLIT=(${LATEST_RELEASE_TAG})
-
- for (( j=0; j<${#THIS_TAG_NAME[@]}; j++ ))
- do
- ## if this value is greater than the latest release, then tag it, if it's lower, then stop, if it's
- ## the same then move on to the next index
- if [[ ${THIS_TAG_NAME[$j]} -gt ${LATEST_RELEASE_TAG_SPLIT[$j]} ]]
- then
- echo "This release tag ${GITHUB_TAG_NAME} is the latest. Tagging it"
- run_git_tag
-
- elif [[ ${THIS_TAG_NAME[$j]} -lt ${LATEST_RELEASE_TAG_SPLIT[$j]} ]]
- then
- continue
- fi
+ # Split the version strings into arrays
+ THIS_TAG_NAME_ARRAY=($(split_string "$THIS_TAG_NAME" "."))
+ LATEST_RELEASE_TAG_ARRAY=($(split_string "$LATEST_RELEASE_TAG" "."))
+
+ # Iterate through the components of the version strings
+ for (( j=0; j<${#THIS_TAG_NAME_ARRAY[@]}; j++ )); do
+ echo "Comparing ${THIS_TAG_NAME_ARRAY[$j]} to ${LATEST_RELEASE_TAG_ARRAY[$j]}"
+ if [[ $((THIS_TAG_NAME_ARRAY[$j])) > $((LATEST_RELEASE_TAG_ARRAY[$j])) ]]; then
+ compare_result="greater"
+ break
+ elif [[ $((THIS_TAG_NAME_ARRAY[$j])) < $((LATEST_RELEASE_TAG_ARRAY[$j])) ]]; then
+ compare_result="lesser"
+ break
+ fi
done
fi
done
-echo "This release tag ${GITHUB_TAG_NAME} is not the latest. Not tagging."
-# if you've gotten this far, then we don't want to run any tags in the next step
-echo "::set-output name=SKIP_TAG::true"
+# Determine the result based on the comparison
+if [[ -z "$compare_result" ]]; then
+ echo "Versions are equal"
+ echo "::set-output name=SKIP_TAG::true"
+elif [[ "$compare_result" == "greater" ]]; then
+ echo "This release tag ${GITHUB_TAG_NAME} is newer than the latest."
+ echo "::set-output name=SKIP_TAG::false"
+ # Add other actions you want to perform for a newer version
+elif [[ "$compare_result" == "lesser" ]]; then
+ echo "This release tag ${GITHUB_TAG_NAME} is older than the latest."
+ echo "This release tag ${GITHUB_TAG_NAME} is not the latest. Not tagging."
+ # if you've gotten this far, then we don't want to run any tags in the next step
+ echo "::set-output name=SKIP_TAG::true"
+fi
diff --git a/tests/unit_tests/fixtures/bash_mock.py b/tests/unit_tests/fixtures/bash_mock.py
new file mode 100644
index 0000000000..91194de6bf
--- /dev/null
+++ b/tests/unit_tests/fixtures/bash_mock.py
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import subprocess
+
+
+class BashMock:
+ @staticmethod
+ def tag_latest_release(tag):
+ bash_command = f"./scripts/tag_latest_release.sh {tag} --dry-run"
+ result = subprocess.run(
+ bash_command,
+ shell=True,
+ capture_output=True,
+ text=True,
+ env={"TEST_ENV": "true"},
+ )
+ return result
+
+ @staticmethod
+ def docker_build_push(tag, branch):
+ bash_command = f"./scripts/docker_build_push.sh {tag}"
+ result = subprocess.run(
+ bash_command,
+ shell=True,
+ capture_output=True,
+ text=True,
+ env={"TEST_ENV": "true", "GITHUB_REF": f"refs/heads/{branch}"},
+ )
+ return result
diff --git a/tests/unit_tests/scripts/docker_build_push_test.py b/tests/unit_tests/scripts/docker_build_push_test.py
new file mode 100644
index 0000000000..9f4c84318e
--- /dev/null
+++ b/tests/unit_tests/scripts/docker_build_push_test.py
@@ -0,0 +1,44 @@
+import re
+import subprocess
+from unittest import mock
+from unittest.mock import patch
+
+import pytest
+
+from tests.unit_tests.fixtures.bash_mock import BashMock
+
+original_run = subprocess.run
+
+
+def wrapped(*args, **kwargs):
+ return original_run(*args, **kwargs)
+
+
+@pytest.mark.parametrize(
+ "tag, expected_output, branch",
+ [
+ ("1.0.0", "LATEST_TAG is master", "master"),
+ ("2.1.0", "LATEST_TAG is master", "master"),
+ ("2.1.1", "LATEST_TAG is latest", "master"),
+ ("3.0.0", "LATEST_TAG is latest", "master"),
+ ("2.1.0rc1", "LATEST_TAG is 2.1.0", "2.1.0"),
+ ("", "LATEST_TAG is foo", "foo"),
+ ("2.1", "LATEST_TAG is 2.1", "2.1"),
+ ("does_not_exist", "LATEST_TAG is does-not-exist", "does_not_exist"),
+ ],
+)
+def test_tag_latest_release(tag, expected_output, branch):
+ with mock.patch(
+ "tests.unit_tests.fixtures.bash_mock.subprocess.run", wraps=wrapped
+ ) as subprocess_mock:
+ result = BashMock.docker_build_push(tag, branch)
+
+ subprocess_mock.assert_called_once_with(
+ f"./scripts/docker_build_push.sh {tag}",
+ shell=True,
+ capture_output=True,
+ text=True,
+ env={"TEST_ENV": "true", "GITHUB_REF": f"refs/heads/{branch}"},
+ )
+
+ assert re.search(expected_output, result.stdout, re.MULTILINE)
diff --git a/tests/unit_tests/scripts/tag_latest_release_test.py b/tests/unit_tests/scripts/tag_latest_release_test.py
new file mode 100644
index 0000000000..7b15a1670a
--- /dev/null
+++ b/tests/unit_tests/scripts/tag_latest_release_test.py
@@ -0,0 +1,49 @@
+import subprocess
+from unittest import mock
+from unittest.mock import patch
+
+import pytest
+
+from tests.unit_tests.fixtures.bash_mock import BashMock
+
+original_run = subprocess.run
+
+
+def wrapped(*args, **kwargs):
+ return original_run(*args, **kwargs)
+
+
+@pytest.mark.parametrize(
+ "tag, expected_output",
+ [
+ ("1.0.0", "This release tag 1.0.0 is older than the latest."),
+ ("2.1.0", "Versions are equal\n::set-output name=SKIP_TAG::true"),
+ ("2.1.1", "This release tag 2.1.1 is newer than the latest."),
+ ("3.0.0", "This release tag 3.0.0 is newer than the latest."),
+ ("2.1.0rc1", "This tag 2.1.0rc1 is not a valid release version. Not tagging."),
+ (
+ "",
+ "Missing tag parameter, usage: ./scripts/tag_latest_release.sh <GITHUB_TAG_NAME>",
+ ),
+ ("2.1", "This tag 2.1 is not a valid release version. Not tagging."),
+ (
+ "does_not_exist",
+ "The tag does_not_exist does not exist. Please use a different tag.\n::set-output name=SKIP_TAG::true",
+ ),
+ ],
+)
+def test_tag_latest_release(tag, expected_output):
+ with mock.patch(
+ "tests.unit_tests.fixtures.bash_mock.subprocess.run", wraps=wrapped
+ ) as subprocess_mock:
+ result = BashMock.tag_latest_release(tag)
+
+ subprocess_mock.assert_called_once_with(
+ f"./scripts/tag_latest_release.sh {tag} --dry-run",
+ shell=True,
+ capture_output=True,
+ text=True,
+ env={"TEST_ENV": "true"},
+ )
+
+ assert expected_output in result.stdout