You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2020/08/26 21:20:11 UTC
[kudu] 05/05: [docker] Support building and pushing multi-arch
images
This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 5e328d38519d656b881219db8087b8eb8af1d4d8
Author: Grant Henke <gr...@apache.org>
AuthorDate: Wed Aug 5 12:18:54 2020 -0500
[docker] Support building and pushing multi-arch images
This change introduces support for building and pushing ARM
and multi-arch Docker images.
Note that building an image for a different architecture than
your machine is painfully slow due to emulation. Improvements
in emulation performance will likely come in the future.
Change-Id: I7f64e4b9591927f160cdf886507cb740578e20b5
Reviewed-on: http://gerrit.cloudera.org:8080/16361
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong <aw...@cloudera.com>
---
docker/docker-build.py | 53 ++++++++++++++++++++++++++++++++++++--------------
1 file changed, 38 insertions(+), 15 deletions(-)
diff --git a/docker/docker-build.py b/docker/docker-build.py
index 5da3847..4b9fb00 100755
--- a/docker/docker-build.py
+++ b/docker/docker-build.py
@@ -67,6 +67,7 @@ ROOT = os.path.abspath(os.path.join(os.path.dirname(ME), ".."))
DEFAULT_OS = 'ubuntu:xenial'
DEFAULT_TARGETS = ['kudu','kudu-python']
DEFAULT_REPOSITORY = 'apache/kudu'
+DEFAULT_ACTION = 'load'
REQUIRED_CPUS = 4
REQUIRED_MEMORY_GIB = 4
@@ -75,6 +76,10 @@ def parse_args():
""" Parses the command-line arguments """
parser = argparse.ArgumentParser(description='Build the Apache Kudu Docker images',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ parser.add_argument('--platforms', nargs='+', choices=[
+ 'linux/amd64', 'linux/arm64', ],
+ help='The platforms to build with. If unspecified, the platform of your '
+ 'build machine will be used.')
parser.add_argument('--bases', nargs='+', default=DEFAULT_OS, choices=[
'centos:6', 'centos:7', 'centos:8',
'debian:jessie', 'debian:stretch',
@@ -90,8 +95,11 @@ def parse_args():
parser.add_argument('--repository', default=DEFAULT_REPOSITORY,
help='The repository string to use when tagging the image')
- parser.add_argument('--publish', action='store_true',
- help='If passed, the tagged images will be pushed to the Docker repository')
+ parser.add_argument('--action', default=DEFAULT_ACTION, choices=['load', 'push'],
+ help='The action to take with the built images. "load" will export the '
+ 'images to docker so they can be used locally. "push" will push the '
+ 'images to the registry.')
+
parser.add_argument('--skip-latest', action='store_true',
help='If passed, skips adding a tag using `-latest` along with the '
'versioned tag')
@@ -119,6 +127,11 @@ def run_command(cmd, opts):
if not opts.dry_run:
subprocess.check_output(cmd, shell=True)
+def use_buildx_builder():
+ ret = subprocess.call(['docker', 'buildx', 'use', 'kudu-builder'])
+ if ret != 0:
+ subprocess.check_output(['docker', 'buildx', 'create', '--name', 'kudu-builder', '--use'])
+
def read_version():
with open(os.path.join(ROOT, 'version.txt'), 'r') as vfile:
return vfile.read().strip()
@@ -174,6 +187,12 @@ def get_full_tag(repository, target, version_tag, os_tag):
full_tag = full_tag[:-1]
return "%s:%s" % (repository, full_tag)
+def platforms_csv(opts):
+ if type(opts.platforms) is list:
+ return ",".join(list(dict.fromkeys(opts.platforms)))
+ else:
+ return opts.platforms
+
def unique_bases(opts):
if type(opts.bases) is list:
return list(dict.fromkeys(opts.bases))
@@ -271,6 +290,10 @@ def main():
# performance, storage management, feature functionality, and security.
# https://docs.docker.com/develop/develop-images/build_enhancements/
os.environ['DOCKER_BUILDKIT'] = '1'
+ # Enable the experimental CLI so we can use `docker buildx` commands.
+ os.environ['DOCKER_CLI_EXPERIMENTAL'] = 'enabled'
+ # Create/Use a buildx builder to support pushing multi-platform builds.
+ use_buildx_builder()
version = read_version()
vcs_ref = read_vcs_ref()
@@ -281,7 +304,11 @@ def main():
print('Bases: %s' % bases)
print('Targets: %s' % targets)
- tags = [] # Keep track of the tags for publishing at the end.
+ if opts.action == 'push' and not is_release_version(version):
+ print('ERROR: Only release versions can be pushed. Found version %s (%s)'
+ % (version, vcs_ref))
+ sys.exit(1)
+
for base in bases:
print('Building targets for %s...' % base)
@@ -289,24 +316,20 @@ def main():
target_tags = build_tags(opts, base, version, vcs_ref, target)
# Build the target.
print('Building %s target...' % target)
- docker_build_cmd = 'docker build'
+ # Note: It isn't currently possible to load multi-arch images into docker,
+ # they can only be pushed to docker hub. This isn't expected to be a long
+ # lived limitation.
+ # https://github.com/docker/buildx/issues/59
+ # https://github.com/moby/moby/pull/38738
+ docker_build_cmd = 'docker buildx build --%s' % opts.action
+ if opts.platforms:
+ docker_build_cmd += ' --platform %s' % platforms_csv(opts)
docker_build_cmd += build_args(base, version, vcs_ref, opts.cache_from)
docker_build_cmd += ' --file %s' % os.path.join(ROOT, 'docker', 'Dockerfile')
docker_build_cmd += ' --target %s' % target
docker_build_cmd += ''.join(' --tag %s' % tag for tag in target_tags)
docker_build_cmd += ' %s' % ROOT
run_command(docker_build_cmd, opts)
- tags.extend(target_tags)
-
- if opts.publish:
- print('Publishing Docker images...')
- if not is_release_version(version):
- print('ERROR: Only release versions can be published. Found version %s (%s)'
- % (version, vcs_ref))
- sys.exit(1)
- for tag in tags:
- push_cmd = "docker push %s" % tag
- run_command(push_cmd, opts)
end_time = datetime.datetime.now()
runtime = end_time - start_time