You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ma...@apache.org on 2024/02/08 20:51:28 UTC
(superset) branch docker_multi_platform_releases updated (3ff4134a72 -> 774077bfc7)
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a change to branch docker_multi_platform_releases
in repository https://gitbox.apache.org/repos/asf/superset.git
discard 3ff4134a72 feat(docker): allow for docker release builds to be multi-platform
new 774077bfc7 feat(docker): allow for docker release builds to be multi-platform
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (3ff4134a72)
\
N -- N -- N refs/heads/docker_multi_platform_releases (774077bfc7)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/docker-release.yml | 14 ++++----------
1 file changed, 4 insertions(+), 10 deletions(-)
(superset) 01/01: feat(docker): allow for docker release builds to be multi-platform
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch docker_multi_platform_releases
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 774077bfc72939a4c302c2a1c0aabc44ba11dfc6
Author: Maxime Beauchemin <ma...@gmail.com>
AuthorDate: Thu Feb 8 12:47:26 2024 -0800
feat(docker): allow for docker release builds to be multi-platform
Allow for release builds to be multi-platform
---
.github/workflows/docker-release.yml | 14 +++-----
scripts/build_docker.py | 34 +++++++++++-------
tests/unit_tests/scripts/docker_build.py | 60 +++++++++++++++++++-------------
3 files changed, 60 insertions(+), 48 deletions(-)
diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml
index 3ba7b6bf42..ae572c910e 100644
--- a/.github/workflows/docker-release.yml
+++ b/.github/workflows/docker-release.yml
@@ -43,14 +43,6 @@ jobs:
strategy:
matrix:
build_preset: ["dev", "lean", "py310", "websocket", "dockerize"]
- platform: ["linux/amd64", "linux/arm64"]
- exclude:
- # disabling because slow! no python wheels for arm/py39 and
- # QEMU is slow!
- - build_preset: "dev"
- platform: "linux/arm64"
- - build_preset: "lean"
- platform: "linux/arm64"
fail-fast: false
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -88,8 +80,10 @@ jobs:
EVENT="release"
fi
pip install click
+ # Make a multi-platform image
./scripts/build_docker.py \
${{ matrix.build_preset }} \
"$EVENT" \
- --build_context_ref "$RELEASE" \
- --platform ${{ matrix.platform }} $FORCE_LATEST
+ --build_context_ref "$RELEASE" $FORCE_LATEST \
+ --platform "linux/arm64"
+ --platform "linux/amd64"
diff --git a/scripts/build_docker.py b/scripts/build_docker.py
index de6b8444f8..095a0f188d 100755
--- a/scripts/build_docker.py
+++ b/scripts/build_docker.py
@@ -79,7 +79,7 @@ def make_docker_tag(l: list[str]) -> str:
def get_docker_tags(
build_preset: str,
- build_platform: str,
+ build_platforms: list[str],
sha: str,
build_context: str,
build_context_ref: str,
@@ -91,17 +91,18 @@ def get_docker_tags(
tags: set[str] = set()
tag_chunks: list[str] = []
- short_build_platform = build_platform.replace("linux/", "").replace("64", "")
-
is_latest = is_latest_release(build_context_ref)
if build_preset != "lean":
# Always add the preset_build name if different from default (lean)
tag_chunks += [build_preset]
- if short_build_platform != "amd":
- # Always a platform indicator if different from default (amd)
- tag_chunks += [short_build_platform]
+ if len(build_platforms) == 1:
+ build_platform = build_platforms[0]
+ short_build_platform = build_platform.replace("linux/", "").replace("64", "")
+ if short_build_platform != "amd":
+ # Always a platform indicator if different from default (amd)
+ tag_chunks += [short_build_platform]
# Always craft a tag for the SHA
tags.add(make_docker_tag([sha] + tag_chunks))
@@ -123,7 +124,7 @@ def get_docker_tags(
def get_docker_command(
build_preset: str,
- build_platform: str,
+ build_platforms: list[str],
is_authenticated: bool,
sha: str,
build_context: str,
@@ -160,7 +161,7 @@ def get_docker_command(
tags = get_docker_tags(
build_preset,
- build_platform,
+ build_platforms,
sha,
build_context,
build_context_ref,
@@ -170,8 +171,14 @@ def get_docker_command(
docker_args = "--load" if not is_authenticated else "--push"
target_argument = f"--target {build_target}" if build_target else ""
- short_build_platform = build_platform.replace("linux/", "").replace("64", "")
- cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
+
+ cache_ref = f"{CACHE_REPO}:{py_ver}"
+ if len(build_platforms) == 1:
+ build_platform = build_platforms[0]
+ short_build_platform = build_platform.replace("linux/", "").replace("64", "")
+ cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
+ platform_arg = "--platform " + ",".join(build_platforms)
+
cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}"
cache_to_arg = (
f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else ""
@@ -187,7 +194,7 @@ def get_docker_command(
{cache_from_arg} \\
{cache_to_arg} \\
{build_arg} \\
- --platform {build_platform} \\
+ {platform_arg} \\
--label sha={sha} \\
--label target={build_target} \\
--label build_trigger={build_context} \\
@@ -206,7 +213,8 @@ def get_docker_command(
@click.option(
"--platform",
type=click.Choice(["linux/arm64", "linux/amd64"]),
- default="linux/amd64",
+ default=["linux/amd64"],
+ multiple=True,
)
@click.option("--build_context_ref", help="a reference to the pr, release or branch")
@click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.")
@@ -217,7 +225,7 @@ def main(
build_preset: str,
build_context: str,
build_context_ref: str,
- platform: str,
+ platform: list[str],
dry_run: bool,
force_latest: bool,
) -> None:
diff --git a/tests/unit_tests/scripts/docker_build.py b/tests/unit_tests/scripts/docker_build.py
index ee9ad66ead..1f3fd708f3 100644
--- a/tests/unit_tests/scripts/docker_build.py
+++ b/tests/unit_tests/scripts/docker_build.py
@@ -56,12 +56,12 @@ def test_is_latest_release(release, expected_bool):
@pytest.mark.parametrize(
- "build_preset, build_platform, sha, build_context, build_context_ref, expected_tags",
+ "build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags",
[
# PRs
(
"lean",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"pull_request",
PR_ID,
@@ -69,7 +69,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"ci",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"pull_request",
PR_ID,
@@ -77,7 +77,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"pull_request",
PR_ID,
@@ -85,7 +85,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"pull_request",
PR_ID,
@@ -97,7 +97,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"pull_request",
PR_ID,
@@ -106,7 +106,7 @@ def test_is_latest_release(release, expected_bool):
# old releases
(
"lean",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"release",
OLD_REL,
@@ -114,7 +114,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"release",
OLD_REL,
@@ -122,7 +122,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"release",
OLD_REL,
@@ -134,7 +134,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"release",
OLD_REL,
@@ -143,7 +143,7 @@ def test_is_latest_release(release, expected_bool):
# new releases
(
"lean",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"release",
NEW_REL,
@@ -156,7 +156,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"release",
NEW_REL,
@@ -164,7 +164,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"release",
NEW_REL,
@@ -177,7 +177,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"release",
NEW_REL,
@@ -191,7 +191,7 @@ def test_is_latest_release(release, expected_bool):
# merge on master
(
"lean",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"push",
"master",
@@ -199,7 +199,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"push",
"master",
@@ -207,7 +207,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"push",
"master",
@@ -219,7 +219,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"push",
"master",
@@ -228,21 +228,21 @@ def test_is_latest_release(release, expected_bool):
],
)
def test_get_docker_tags(
- build_preset, build_platform, sha, build_context, build_context_ref, expected_tags
+ build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags
):
tags = docker_utils.get_docker_tags(
- build_preset, build_platform, sha, build_context, build_context_ref
+ build_preset, build_platforms, sha, build_context, build_context_ref
)
for tag in expected_tags:
assert tag in tags
@pytest.mark.parametrize(
- "build_preset, build_platform, is_authenticated, sha, build_context, build_context_ref, contains",
+ "build_preset, build_platforms, is_authenticated, sha, build_context, build_context_ref, contains",
[
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
True,
SHA,
"push",
@@ -251,18 +251,28 @@ def test_get_docker_tags(
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
False,
SHA,
"push",
"master",
["--load", f"-t {REPO}:master-dev "],
),
+ # multi-platform
+ (
+ "lean",
+ ["linux/arm64", "linux/amd64"],
+ True,
+ SHA,
+ "push",
+ "master",
+ [f"--platform linux/arm64,linux/amd64"],
+ ),
],
)
def test_get_docker_command(
build_preset,
- build_platform,
+ build_platforms,
is_authenticated,
sha,
build_context,
@@ -271,7 +281,7 @@ def test_get_docker_command(
):
cmd = docker_utils.get_docker_command(
build_preset,
- build_platform,
+ build_platforms,
is_authenticated,
sha,
build_context,