You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2022/04/26 15:06:25 UTC
[couchdb-ci] 02/07: Use buildx plugin for multi-arch images
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb-ci.git
commit 946c094606891ac14e3e1f861e2ba443dcaf45bf
Author: Adam Kocoloski <ko...@apache.org>
AuthorDate: Mon Feb 7 22:04:54 2022 -0500
Use buildx plugin for multi-arch images
---
README.md | 41 +++++++++++++++++++++++++++--------------
build.sh | 31 +++++--------------------------
dockerfiles/debian-bullseye | 5 +----
dockerfiles/debian-buster | 4 +---
dockerfiles/debian-stretch | 5 +----
5 files changed, 35 insertions(+), 51 deletions(-)
diff --git a/README.md b/README.md
index 67973be..50a35a2 100644
--- a/README.md
+++ b/README.md
@@ -31,6 +31,12 @@ These images are used by [Apache Jenkins CI](https://ci-couchdb.apache.org/blue/
For those OSes that support Docker, we run builds inside of Docker containers. These containers are built using the `build.sh` command at the root level.
+## Authenticating to Docker Hub
+
+1. You need a Docker Cloud account with access to the `apache` organization to upload new images. Ask the CouchDB PMC for assistance with this.
+2. `export DOCKER_ID_USER="username"`
+3. `docker login -u "username"` and enter your password.
+
## Building a "platform image"
The platform images include all of the build dependencies necessary to build and full test CouchDB on a given OS/version/architecture combination.
@@ -59,31 +65,38 @@ the [kerl](https://github.com/kerl/kerl) build system, and installs them to
`/usr/local/kerl` for activation before builds. This version is intended for use
in standard CI runs, such as for pull requests.
-# Building a cross-architecture Docker image
+## Building a cross-architecture Docker image
-This only works from an `x86_64` build host.
-
-First, configure your machine with the correct dependencies to build multi-arch binaries:
+We can use Docker's
+[Buildx](https://docs.docker.com/buildx/working-with-buildx/) plugin to generate
+multi-architecture container images with a single command invocation. Docker
+Desktop ships with buildx support, but you'll need to create a new builder to
+use it:
```
-docker run --privileged --rm tonistiigi/binfmt --install all
+docker buildx create --use
```
-This is a one-time setup step. This docker container run will install the correct qemu static binaries necessary for running foreign architecture binaries on your host machine. It includes special magic to ensure `sudo` works correctly inside a container, too.
-
-Then, override the `CONTAINERARCH` environment variable when starting `build.sh`:
-
+Then, add the `--platform` flag to spin up parallel builders for each desired
+architecture. For example:
```
-CONTAINERARCH=aarch64 ./build.sh platform debian-stretch
+buildargs="--platform linux/amd64,linux/arm64,linux/ppc64le --push" ./build.sh platform debian-bullseye
```
+will build **and upload** a new multi-arch container image to the registory.
+Currently `docker images` can only accept single-platform images, so this is one
+downside of the simplified build approach. Omitting the `--push` option will
+just leave the build result in the build cache, which isn't terribly useful.
+
## Publishing a container
-1. You need a Docker Cloud account with access to the `apache` organization. Ask the CouchDB PMC for assistance with this.
-2. `export DOCKER_ID_USER="username"`
-3. `docker login -u "username"` and enter your password.
-4. `./build.sh platform-upload <distro>-<version>` just as above.
+If you built a single-architecture container image and did not supply `--push`
+as a build arg to upload it automatically you can upload the image using
+
+```
+./build.sh platform-upload <distro>-<version>
+```
---
diff --git a/build.sh b/build.sh
index 611d2fc..ca142c9 100755
--- a/build.sh
+++ b/build.sh
@@ -46,9 +46,6 @@ DEBIANS="debian-stretch debian-buster debian-bullseye"
UBUNTUS="ubuntu-bionic ubuntu-focal"
CENTOSES="centos-7 centos-8"
ERLANGALL_BASE="debian-bullseye"
-XPLAT_BASE="debian-buster"
-# XPLAT_ARCHES="arm64v8 ppc64le"
-XPLAT_ARCHES="arm64v8"
BINTRAY_API="https://api.bintray.com"
PASSED_BUILDARGS="$buildargs"
@@ -67,11 +64,6 @@ check-envs() {
then
buildargs="$buildargs --build-arg elixirversion=${ELIXIRVERSION} "
fi
- if [ ! -z "${CONTAINERARCH}" ]
- then
- buildargs="$buildargs --build-arg containerarch=${CONTAINERARCH} "
- CONTAINERARCH="${CONTAINERARCH}-"
- fi
}
split-os-ver() {
@@ -87,11 +79,11 @@ build-base-platform() {
split-os-ver $1
# invoke as build-base <plat>
# base images never get JavaScript, nor Erlang
- docker build -f dockerfiles/${os}-${version} \
+ docker buildx build -f dockerfiles/${os}-${version} \
--build-arg js=nojs \
--build-arg erlang=noerlang \
$buildargs \
- --tag apache/couchdbci-${os}:${CONTAINERARCH}${version}-base \
+ --tag apache/couchdbci-${os}:${version}-base \
${SCRIPTPATH}
}
@@ -122,10 +114,10 @@ build-platform() {
find-erlang-version $1
pull-os-image $1
split-os-ver $1
- docker build -f dockerfiles/${os}-${version} \
+ docker buildx build -f dockerfiles/${os}-${version} \
$buildargs \
--no-cache \
- --tag apache/couchdbci-${os}:${CONTAINERARCH}${version}-erlang-${ERLANGVERSION} \
+ --tag apache/couchdbci-${os}:${version}-erlang-${ERLANGVERSION} \
${SCRIPTPATH}
unset ERLANGVERSION
}
@@ -146,7 +138,7 @@ upload-platform() {
find-erlang-version $1
check-envs
split-os-ver $1
- docker push apache/couchdbci-${os}:${CONTAINERARCH}${version}-erlang-${ERLANGVERSION}
+ docker push apache/couchdbci-${os}:${version}-erlang-${ERLANGVERSION}
}
build-test-couch() {
@@ -200,22 +192,12 @@ case "$1" in
shift
build-platform $1
;;
- platform-foreign)
- # makes only foreign arch platforms
- shift
- for arch in $XPLAT_ARCHES; do
- CONTAINERARCH=$arch build-platform $XPLAT_BASE
- done
- ;;
platform-all)
# build all platforms with JS and Erlang support
shift
for plat in $DEBIANS $UBUNTUS $CENTOSES; do
build-platform $plat $*
done
- for arch in $XPLAT_ARCHES; do
- CONTAINERARCH=$arch build-platform $XPLAT_BASE
- done
ERLANGVERSION=all build-platform $ERLANGALL_BASE
;;
platform-upload)
@@ -227,9 +209,6 @@ case "$1" in
for plat in $DEBIANS $UBUNTUS $CENTOSES; do
upload-platform $plat $*
done
- for arch in $XPLAT_ARCHES; do
- CONTAINERARCH=$arch upload-platform $XPLAT_BASE $*
- done
ERLANGVERSION=all upload-platform $ERLANGALL_BASE
;;
couch)
diff --git a/dockerfiles/debian-bullseye b/dockerfiles/debian-bullseye
index 690b2eb..ca91c08 100644
--- a/dockerfiles/debian-bullseye
+++ b/dockerfiles/debian-bullseye
@@ -17,10 +17,7 @@
# NOTE: These are intended to be built using the arguments as
# described in ../build.sh. See that script for more details.
-# Support multi-platform builds with a single Dockerfile
-ARG containerarch=amd64
-
-FROM $containerarch/debian:bullseye
+FROM debian:bullseye
# Choose whether to install SpiderMonkey 1.8.5, default yes
ARG js=js
diff --git a/dockerfiles/debian-buster b/dockerfiles/debian-buster
index 58c2b95..8621fbd 100644
--- a/dockerfiles/debian-buster
+++ b/dockerfiles/debian-buster
@@ -18,9 +18,7 @@
# described in ../build.sh. See that script for more details.
# Support multi-platform builds with a single Dockerfile
-ARG containerarch=amd64
-
-FROM $containerarch/debian:buster
+FROM debian:buster
# Choose whether to install SpiderMonkey 1.8.5, default yes
ARG js=js
diff --git a/dockerfiles/debian-stretch b/dockerfiles/debian-stretch
index 23a4be5..10f3ca5 100644
--- a/dockerfiles/debian-stretch
+++ b/dockerfiles/debian-stretch
@@ -17,10 +17,7 @@
# NOTE: These are intended to be built using the arguments as
# described in ../build.sh. See that script for more details.
-# Support multi-platform builds with a single Dockerfile
-ARG containerarch=amd64
-
-FROM $containerarch/debian:stretch
+FROM debian:stretch
# Choose whether to install SpiderMonkey 1.8.5, default yes
ARG js=js