You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ma...@apache.org on 2024/02/08 20:47:50 UTC
(superset) 01/01: feat(docker): allow for docker release builds to be multi-platform
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch docker_multi_platform_releases
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 3ff4134a72c873cf779db7bf98120729aa81581e
Author: Maxime Beauchemin <ma...@gmail.com>
AuthorDate: Thu Feb 8 12:47:26 2024 -0800
feat(docker): allow for docker release builds to be multi-platform
---
scripts/build_docker.py | 34 +++++++++++-------
tests/unit_tests/scripts/docker_build.py | 60 +++++++++++++++++++-------------
2 files changed, 56 insertions(+), 38 deletions(-)
diff --git a/scripts/build_docker.py b/scripts/build_docker.py
index de6b8444f8..095a0f188d 100755
--- a/scripts/build_docker.py
+++ b/scripts/build_docker.py
@@ -79,7 +79,7 @@ def make_docker_tag(l: list[str]) -> str:
def get_docker_tags(
build_preset: str,
- build_platform: str,
+ build_platforms: list[str],
sha: str,
build_context: str,
build_context_ref: str,
@@ -91,17 +91,18 @@ def get_docker_tags(
tags: set[str] = set()
tag_chunks: list[str] = []
- short_build_platform = build_platform.replace("linux/", "").replace("64", "")
-
is_latest = is_latest_release(build_context_ref)
if build_preset != "lean":
# Always add the preset_build name if different from default (lean)
tag_chunks += [build_preset]
- if short_build_platform != "amd":
- # Always a platform indicator if different from default (amd)
- tag_chunks += [short_build_platform]
+ if len(build_platforms) == 1:
+ build_platform = build_platforms[0]
+ short_build_platform = build_platform.replace("linux/", "").replace("64", "")
+ if short_build_platform != "amd":
+ # Always a platform indicator if different from default (amd)
+ tag_chunks += [short_build_platform]
# Always craft a tag for the SHA
tags.add(make_docker_tag([sha] + tag_chunks))
@@ -123,7 +124,7 @@ def get_docker_tags(
def get_docker_command(
build_preset: str,
- build_platform: str,
+ build_platforms: list[str],
is_authenticated: bool,
sha: str,
build_context: str,
@@ -160,7 +161,7 @@ def get_docker_command(
tags = get_docker_tags(
build_preset,
- build_platform,
+ build_platforms,
sha,
build_context,
build_context_ref,
@@ -170,8 +171,14 @@ def get_docker_command(
docker_args = "--load" if not is_authenticated else "--push"
target_argument = f"--target {build_target}" if build_target else ""
- short_build_platform = build_platform.replace("linux/", "").replace("64", "")
- cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
+
+ cache_ref = f"{CACHE_REPO}:{py_ver}"
+ if len(build_platforms) == 1:
+ build_platform = build_platforms[0]
+ short_build_platform = build_platform.replace("linux/", "").replace("64", "")
+ cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
+ platform_arg = "--platform " + ",".join(build_platforms)
+
cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}"
cache_to_arg = (
f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else ""
@@ -187,7 +194,7 @@ def get_docker_command(
{cache_from_arg} \\
{cache_to_arg} \\
{build_arg} \\
- --platform {build_platform} \\
+ {platform_arg} \\
--label sha={sha} \\
--label target={build_target} \\
--label build_trigger={build_context} \\
@@ -206,7 +213,8 @@ def get_docker_command(
@click.option(
"--platform",
type=click.Choice(["linux/arm64", "linux/amd64"]),
- default="linux/amd64",
+ default=["linux/amd64"],
+ multiple=True,
)
@click.option("--build_context_ref", help="a reference to the pr, release or branch")
@click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.")
@@ -217,7 +225,7 @@ def main(
build_preset: str,
build_context: str,
build_context_ref: str,
- platform: str,
+ platform: list[str],
dry_run: bool,
force_latest: bool,
) -> None:
diff --git a/tests/unit_tests/scripts/docker_build.py b/tests/unit_tests/scripts/docker_build.py
index ee9ad66ead..1f3fd708f3 100644
--- a/tests/unit_tests/scripts/docker_build.py
+++ b/tests/unit_tests/scripts/docker_build.py
@@ -56,12 +56,12 @@ def test_is_latest_release(release, expected_bool):
@pytest.mark.parametrize(
- "build_preset, build_platform, sha, build_context, build_context_ref, expected_tags",
+ "build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags",
[
# PRs
(
"lean",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"pull_request",
PR_ID,
@@ -69,7 +69,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"ci",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"pull_request",
PR_ID,
@@ -77,7 +77,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"pull_request",
PR_ID,
@@ -85,7 +85,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"pull_request",
PR_ID,
@@ -97,7 +97,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"pull_request",
PR_ID,
@@ -106,7 +106,7 @@ def test_is_latest_release(release, expected_bool):
# old releases
(
"lean",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"release",
OLD_REL,
@@ -114,7 +114,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"release",
OLD_REL,
@@ -122,7 +122,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"release",
OLD_REL,
@@ -134,7 +134,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"release",
OLD_REL,
@@ -143,7 +143,7 @@ def test_is_latest_release(release, expected_bool):
# new releases
(
"lean",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"release",
NEW_REL,
@@ -156,7 +156,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"release",
NEW_REL,
@@ -164,7 +164,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"release",
NEW_REL,
@@ -177,7 +177,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"release",
NEW_REL,
@@ -191,7 +191,7 @@ def test_is_latest_release(release, expected_bool):
# merge on master
(
"lean",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"push",
"master",
@@ -199,7 +199,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"push",
"master",
@@ -207,7 +207,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"push",
"master",
@@ -219,7 +219,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"push",
"master",
@@ -228,21 +228,21 @@ def test_is_latest_release(release, expected_bool):
],
)
def test_get_docker_tags(
- build_preset, build_platform, sha, build_context, build_context_ref, expected_tags
+ build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags
):
tags = docker_utils.get_docker_tags(
- build_preset, build_platform, sha, build_context, build_context_ref
+ build_preset, build_platforms, sha, build_context, build_context_ref
)
for tag in expected_tags:
assert tag in tags
@pytest.mark.parametrize(
- "build_preset, build_platform, is_authenticated, sha, build_context, build_context_ref, contains",
+ "build_preset, build_platforms, is_authenticated, sha, build_context, build_context_ref, contains",
[
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
True,
SHA,
"push",
@@ -251,18 +251,28 @@ def test_get_docker_tags(
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
False,
SHA,
"push",
"master",
["--load", f"-t {REPO}:master-dev "],
),
+ # multi-platform
+ (
+ "lean",
+ ["linux/arm64", "linux/amd64"],
+ True,
+ SHA,
+ "push",
+ "master",
+ [f"--platform linux/arm64,linux/amd64"],
+ ),
],
)
def test_get_docker_command(
build_preset,
- build_platform,
+ build_platforms,
is_authenticated,
sha,
build_context,
@@ -271,7 +281,7 @@ def test_get_docker_command(
):
cmd = docker_utils.get_docker_command(
build_preset,
- build_platform,
+ build_platforms,
is_authenticated,
sha,
build_context,