You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ma...@apache.org on 2024/02/08 20:47:49 UTC
(superset) branch docker_multi_platform_releases created (now 3ff4134a72)
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a change to branch docker_multi_platform_releases
in repository https://gitbox.apache.org/repos/asf/superset.git
at 3ff4134a72 feat(docker): allow for docker release builds to be multi-platform
This branch includes the following new commits:
new 3ff4134a72 feat(docker): allow for docker release builds to be multi-platform
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(superset) 01/01: feat(docker): allow for docker release builds to be multi-platform
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch docker_multi_platform_releases
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 3ff4134a72c873cf779db7bf98120729aa81581e
Author: Maxime Beauchemin <ma...@gmail.com>
AuthorDate: Thu Feb 8 12:47:26 2024 -0800
feat(docker): allow for docker release builds to be multi-platform
---
scripts/build_docker.py | 34 +++++++++++-------
tests/unit_tests/scripts/docker_build.py | 60 +++++++++++++++++++-------------
2 files changed, 56 insertions(+), 38 deletions(-)
diff --git a/scripts/build_docker.py b/scripts/build_docker.py
index de6b8444f8..095a0f188d 100755
--- a/scripts/build_docker.py
+++ b/scripts/build_docker.py
@@ -79,7 +79,7 @@ def make_docker_tag(l: list[str]) -> str:
def get_docker_tags(
build_preset: str,
- build_platform: str,
+ build_platforms: list[str],
sha: str,
build_context: str,
build_context_ref: str,
@@ -91,17 +91,18 @@ def get_docker_tags(
tags: set[str] = set()
tag_chunks: list[str] = []
- short_build_platform = build_platform.replace("linux/", "").replace("64", "")
-
is_latest = is_latest_release(build_context_ref)
if build_preset != "lean":
# Always add the preset_build name if different from default (lean)
tag_chunks += [build_preset]
- if short_build_platform != "amd":
- # Always a platform indicator if different from default (amd)
- tag_chunks += [short_build_platform]
+ if len(build_platforms) == 1:
+ build_platform = build_platforms[0]
+ short_build_platform = build_platform.replace("linux/", "").replace("64", "")
+ if short_build_platform != "amd":
+ # Always a platform indicator if different from default (amd)
+ tag_chunks += [short_build_platform]
# Always craft a tag for the SHA
tags.add(make_docker_tag([sha] + tag_chunks))
@@ -123,7 +124,7 @@ def get_docker_tags(
def get_docker_command(
build_preset: str,
- build_platform: str,
+ build_platforms: list[str],
is_authenticated: bool,
sha: str,
build_context: str,
@@ -160,7 +161,7 @@ def get_docker_command(
tags = get_docker_tags(
build_preset,
- build_platform,
+ build_platforms,
sha,
build_context,
build_context_ref,
@@ -170,8 +171,14 @@ def get_docker_command(
docker_args = "--load" if not is_authenticated else "--push"
target_argument = f"--target {build_target}" if build_target else ""
- short_build_platform = build_platform.replace("linux/", "").replace("64", "")
- cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
+
+ cache_ref = f"{CACHE_REPO}:{py_ver}"
+ if len(build_platforms) == 1:
+ build_platform = build_platforms[0]
+ short_build_platform = build_platform.replace("linux/", "").replace("64", "")
+ cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
+ platform_arg = "--platform " + ",".join(build_platforms)
+
cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}"
cache_to_arg = (
f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else ""
@@ -187,7 +194,7 @@ def get_docker_command(
{cache_from_arg} \\
{cache_to_arg} \\
{build_arg} \\
- --platform {build_platform} \\
+ {platform_arg} \\
--label sha={sha} \\
--label target={build_target} \\
--label build_trigger={build_context} \\
@@ -206,7 +213,8 @@ def get_docker_command(
@click.option(
"--platform",
type=click.Choice(["linux/arm64", "linux/amd64"]),
- default="linux/amd64",
+ default=["linux/amd64"],
+ multiple=True,
)
@click.option("--build_context_ref", help="a reference to the pr, release or branch")
@click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.")
@@ -217,7 +225,7 @@ def main(
build_preset: str,
build_context: str,
build_context_ref: str,
- platform: str,
+ platform: list[str],
dry_run: bool,
force_latest: bool,
) -> None:
diff --git a/tests/unit_tests/scripts/docker_build.py b/tests/unit_tests/scripts/docker_build.py
index ee9ad66ead..1f3fd708f3 100644
--- a/tests/unit_tests/scripts/docker_build.py
+++ b/tests/unit_tests/scripts/docker_build.py
@@ -56,12 +56,12 @@ def test_is_latest_release(release, expected_bool):
@pytest.mark.parametrize(
- "build_preset, build_platform, sha, build_context, build_context_ref, expected_tags",
+ "build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags",
[
# PRs
(
"lean",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"pull_request",
PR_ID,
@@ -69,7 +69,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"ci",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"pull_request",
PR_ID,
@@ -77,7 +77,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"pull_request",
PR_ID,
@@ -85,7 +85,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"pull_request",
PR_ID,
@@ -97,7 +97,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"pull_request",
PR_ID,
@@ -106,7 +106,7 @@ def test_is_latest_release(release, expected_bool):
# old releases
(
"lean",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"release",
OLD_REL,
@@ -114,7 +114,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"release",
OLD_REL,
@@ -122,7 +122,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"release",
OLD_REL,
@@ -134,7 +134,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"release",
OLD_REL,
@@ -143,7 +143,7 @@ def test_is_latest_release(release, expected_bool):
# new releases
(
"lean",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"release",
NEW_REL,
@@ -156,7 +156,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"release",
NEW_REL,
@@ -164,7 +164,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"release",
NEW_REL,
@@ -177,7 +177,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"release",
NEW_REL,
@@ -191,7 +191,7 @@ def test_is_latest_release(release, expected_bool):
# merge on master
(
"lean",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"push",
"master",
@@ -199,7 +199,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"push",
"master",
@@ -207,7 +207,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/arm64",
+ ["linux/arm64"],
SHA,
"push",
"master",
@@ -219,7 +219,7 @@ def test_is_latest_release(release, expected_bool):
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
SHA,
"push",
"master",
@@ -228,21 +228,21 @@ def test_is_latest_release(release, expected_bool):
],
)
def test_get_docker_tags(
- build_preset, build_platform, sha, build_context, build_context_ref, expected_tags
+ build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags
):
tags = docker_utils.get_docker_tags(
- build_preset, build_platform, sha, build_context, build_context_ref
+ build_preset, build_platforms, sha, build_context, build_context_ref
)
for tag in expected_tags:
assert tag in tags
@pytest.mark.parametrize(
- "build_preset, build_platform, is_authenticated, sha, build_context, build_context_ref, contains",
+ "build_preset, build_platforms, is_authenticated, sha, build_context, build_context_ref, contains",
[
(
"lean",
- "linux/amd64",
+ ["linux/amd64"],
True,
SHA,
"push",
@@ -251,18 +251,28 @@ def test_get_docker_tags(
),
(
"dev",
- "linux/amd64",
+ ["linux/amd64"],
False,
SHA,
"push",
"master",
["--load", f"-t {REPO}:master-dev "],
),
+ # multi-platform
+ (
+ "lean",
+ ["linux/arm64", "linux/amd64"],
+ True,
+ SHA,
+ "push",
+ "master",
+ [f"--platform linux/arm64,linux/amd64"],
+ ),
],
)
def test_get_docker_command(
build_preset,
- build_platform,
+ build_platforms,
is_authenticated,
sha,
build_context,
@@ -271,7 +281,7 @@ def test_get_docker_command(
):
cmd = docker_utils.get_docker_command(
build_preset,
- build_platform,
+ build_platforms,
is_authenticated,
sha,
build_context,