You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2026/10/12 05:14:01 UTC
(couchdb-pkg) 01/01: Cleanup package builds and use remote buildx nodes
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch cleanup-and-buildx-remotes-for-pkgs
in repository https://gitbox.apache.org/repos/asf/couchdb-pkg.git
commit 1c4560d6ea204de43cc579b4060579b05eac6285
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Mon Oct 12 01:09:52 2026 -0400
Cleanup package builds and use remote buildx nodes
Cleanup some makefile targets (remove the silly redundancy), and also
build package on remote buildx instances, based on architecture/os
combos. That's how CI images are built. And up until now package were
still built on my (host) laptop using emulation which was very slow.
The setup is something like
```
docker buildx rm multiarch || true
docker buildx create --name multiarch --driver docker-container --platform linux/arm64 ssh://ubuntu-nc-arm64-12
docker buildx create --append --name multiarch --driver docker-container --platform linux/amd64 ssh://ubuntu-fsn-1-09
docker buildx create --append --name multiarch --driver docker-container --platform linux/s390x ssh://linux1
docker buildx create --append --name multiarch --driver docker-container --platform linux/ppc64le ssh://ubuntu
```
Re-bootstrap after OS updates
```
docker buildx inspect --bootstrap
```
---
.dockerignore | 14 +++++++
Dockerfile | 36 +++++++++++++++++
Makefile | 121 +++++-----------------------------------------------------
README.md | 57 +++++++++++++++------------
build.sh | 43 ++++++++++-----------
5 files changed, 113 insertions(+), 158 deletions(-)
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..bcdd323
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,14 @@
+# Keep only the package sources and CouchDB tarball
+.git
+.idea
+archive
+pkgs
+pkgs.bak*
+*.bak
+repo/db
+repo/dists
+repo/pool
+parts
+prime
+stage
+*.snap
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..2c4cb4d
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,36 @@
+# check=skip=InvalidDefaultArgInFrom
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+# Build CouchDB packages for an {os,arch} combo inside the matching couchdb-ci image.
+# Run from build.sh as:
+#
+# docker buildx build --platform linux/<arch> --output type=local,dest=pkgs/couch ...
+#
+# The reuslt is the built package(s). Then the local exporter saves
+# them into pkgs/couch/$dist on the host. The build itself run on
+# whatever node of the current builder supports combo (possibly
+# remote).
+
+# IMAGE always passed in by build.sh. No good default for it, so added
+# lint in line 1 so it stops complaining about.
+ARG IMAGE
+FROM ${IMAGE} AS build
+ARG TARGET
+ARG COUCHTARBALL
+USER root
+WORKDIR /home/jenkins/couchdb-pkg
+COPY . .
+RUN make copy-couch ${TARGET} COUCHTARBALL=${COUCHTARBALL}
+
+FROM scratch
+COPY --from=build /home/jenkins/couchdb-pkg/pkgs/couch/ /
diff --git a/Makefile b/Makefile
index 06490fb..ee11b47 100644
--- a/Makefile
+++ b/Makefile
@@ -21,13 +21,9 @@ export DEBEMAIL=dev@couchdb.apache.org
# Default package directory (over-written for RPM based builds)
PKGDIR=$(COUCHDIR)
-ifeq ($(shell arch),aarch64)
-PKGARCH=arm64
-else ifeq ($(shell arch),arm64v8)
-PKGARCH=arm64
-else
+# RPMs go into the RPMS/<arch> dir where where <arch> is `uname -m`
+# (x86_64, aarch64, ppc64le, ...). DEB builds figure out their own arch
PKGARCH=$(shell arch)
-endif
# Overridden for targets with SM60 in per-target env vars below.
SPIDERMONKEY=couch-libmozjs185-1.0
@@ -42,6 +38,14 @@ JAVA_RUNTIME=java21-runtime-headless | java21-runtime
all:
make `bin/detect-target.sh`
+# arch specific prefixes arm64-ubuntu-noble are aliases for plain
+# target
+arm64-%: % ; @:
+arm64v8-%: % ; @:
+aarch64-%: % ; @:
+ppc64le-%: % ; @:
+s390x-%: % ; @:
+
# Debian default
debian: sm-ver-debian find-couch-dist copy-debian update-changelog dpkg lintian copy-pkgs
@@ -53,30 +57,6 @@ debian-bookworm: SPIDERMONKEY_DEV=libmozjs-78-dev
debian-bookworm: SM_VER=78
debian-bookworm: bookworm
-arm64-debian-bookworm: aarch64-debian-bookworm
-arm64v8-debian-bookworm: aarch64-debian-bookworm
-aarch64-debian-bookworm: PLATFORM=bookworm
-aarch64-debian-bookworm: DIST=debian-bookworm
-aarch64-debian-bookworm: SPIDERMONKEY=libmozjs-78-0
-aarch64-debian-bookworm: SPIDERMONKEY_DEV=libmozjs-78-dev
-aarch64-debian-bookworm: SM_VER=78
-aarch64-debian-bookworm: bookworm
-
-ppc64le-debian-bookworm: PLATFORM=bookworm
-ppc64le-debian-bookworm: DIST=debian-bookworm
-ppc64le-debian-bookworm: SPIDERMONKEY=libmozjs-78-0
-ppc64le-debian-bookworm: SPIDERMONKEY_DEV=libmozjs-78-dev
-ppc64le-debian-bookworm: SM_VER=78
-ppc64le-debian-bookworm: bookworm
-
-s390x-debian-bookworm: PLATFORM=bookworm
-s390x-debian-bookworm: DIST=debian-bookworm
-s390x-debian-bookworm: SPIDERMONKEY=libmozjs-78-0
-s390x-debian-bookworm: SPIDERMONKEY_DEV=libmozjs-78-dev
-s390x-debian-bookworm: SM_VER=78
-s390x-debian-bookworm: bookworm
-
-
bookworm: debian
# Debian 13 - trixie
@@ -88,32 +68,6 @@ debian-trixie: SM_VER=128
debian-trixie: JAVA_RUNTIME=java21-runtime-headless | java21-runtime
debian-trixie: trixie
-arm64-debian-trixie: aarch64-debian-trixie
-arm64v8-debian-trixie: aarch64-debian-trixie
-aarch64-debian-trixie: PLATFORM=trixie
-aarch64-debian-trixie: DIST=debian-trixie
-aarch64-debian-trixie: SPIDERMONKEY=libmozjs-128-0
-aarch64-debian-trixie: SPIDERMONKEY_DEV=libmozjs-128-dev
-aarch64-debian-trixie: SM_VER=128
-aarch64-debian-trixie: JAVA_RUNTIME=java21-runtime-headless | java21-runtime
-aarch64-debian-trixie: trixie
-
-ppc64le-debian-trixie: PLATFORM=trixie
-ppc64le-debian-trixie: DIST=debian-trixie
-ppc64le-debian-trixie: SPIDERMONKEY=libmozjs-128-0
-ppc64le-debian-trixie: SPIDERMONKEY_DEV=libmozjs-128-dev
-ppc64le-debian-trixie: SM_VER=128
-ppc64le-debian-trixie: JAVA_RUNTIME=java21-runtime-headless | java21-runtime
-ppc64le-debian-trixie: trixie
-
-s390x-debian-trixie: PLATFORM=trixie
-s390x-debian-trixie: DIST=debian-trixie
-s390x-debian-trixie: SPIDERMONKEY=libmozjs-128-0
-s390x-debian-trixie: SPIDERMONKEY_DEV=libmozjs-128-dev
-s390x-debian-trixie: SM_VER=128
-s390x-debian-trixie: JAVA_RUNTIME=java21-runtime-headless | java21-runtime
-s390x-debian-trixie: trixie
-
trixie: debian
# Ubuntu 22.04 (Jammy)
@@ -125,11 +79,6 @@ ubuntu-jammy: SM_VER=78
ubuntu-jammy: jammy
jammy: debian
-s390x-ubuntu-jammy: ubuntu-jammy
-arm64-ubuntu-jammy: ubuntu-jammy
-aarch64-ubuntu-jammy: ubuntu-jammy
-ppc64le-ubuntu-jammy: ubuntu-jammy
-
# Ubuntu 24.04 (Noble)
ubuntu-noble: PLATFORM=noble
ubuntu-noble: DIST=ubuntu-noble
@@ -139,11 +88,6 @@ ubuntu-noble: SM_VER=115
ubuntu-noble: noble
noble: debian
-s390x-ubuntu-noble: ubuntu-noble
-arm64-ubuntu-noble: ubuntu-noble
-aarch64-ubuntu-noble: ubuntu-noble
-ppc64le-ubuntu-noble: ubuntu-noble
-
# Ubuntu 26.04 (Resolute)
ubuntu-resolute: PLATFORM=resolute
ubuntu-resolute: DIST=ubuntu-resolute
@@ -153,11 +97,6 @@ ubuntu-resolute: SM_VER=128
ubuntu-resolute: resolute
resolute: debian
-s390x-ubuntu-resolute: ubuntu-resolute
-arm64-ubuntu-resolute: ubuntu-resolute
-aarch64-ubuntu-resolute: ubuntu-resolute
-ppc64le-ubuntu-resolute: ubuntu-resolute
-
# RPM default
centos: PKGDIR=../rpmbuild/RPMS/$(PKGARCH)
centos: find-couch-dist link-couch-dist build-rpm copy-pkgs
@@ -186,10 +125,6 @@ almalinux-8: centos-8
almalinux-8.8: centos-8
almalinux-8.9: centos-8
almalinux-8.10: centos-8
-aarch64-almalinux-8.10: PKGARCH=aarch64
-aarch64-almalinux-8.10: centos-8
-aarch64-almalinux-8: PKGARCH=aarch64
-aarch64-almalinux-8: centos-8
# Almalinux 9 is a CentOS 9 alias
almalinux-9: centos-9
@@ -199,47 +134,11 @@ almalinux-9.5: centos-9
almalinux-9.6: centos-9
almalinux-9.7: centos-9
almalinux-9.8: centos-9
-aarch64-almalinux-9.4: PKGARCH=aarch64
-aarch64-almalinux-9.4: centos-9
-aarch64-almalinux-9.5: PKGARCH=aarch64
-aarch64-almalinux-9.5: centos-9
-aarch64-almalinux-9.6: PKGARCH=aarch64
-aarch64-almalinux-9.6: centos-9
-aarch64-almalinux-9.7: PKGARCH=aarch64
-aarch64-almalinux-9.7: centos-9
-aarch64-almalinux-9.8: PKGARCH=aarch64
-aarch64-almalinux-9.8: centos-9
-aarch64-almalinux-9: PKGARCH=aarch64
-aarch64-almalinux-9: centos-9
-# s390x RHEL 8 clone based
-s390x-centos-8: centos-8
-ppc64le-centos-8: centos-8
-# s390x RHEL 9 clone based
-s390x-centos-9: centos-9
-
-arm64-centos-9: PKGARCH=aarch64
-arm64-centos-9: centos-9
-ppc64le-centos-9: centos-9
# Almalinux 10 is a CentOS 10 alias
almalinux-10: centos-10
almalinux-10.1: centos-10
almalinux-10.2: centos-10
-aarch64-almalinux-10: PKGARCH=aarch64
-aarch64-almalinux-10: centos-10
-aarch64-almalinux-10.1: PKGARCH=aarch64
-aarch64-almalinux-10.1: centos-10
-aarch64-almalinux-10.2: PKGARCH=aarch64
-aarch64-almalinux-10.2: centos-10
-# s390x RHEL 10 clone based
-s390x-centos-10: centos-10
-ppc64le-centos-10: centos-10
-# s390x RHEL 10 clone based
-s390x-centos-10: centos-10
-
-arm64-centos-10: PKGARCH=aarch64
-arm64-centos-10: centos-10
-ppc64le-centos-10: centos-10
# aarch64 RHEL-based
aarch64-rhel: DIST=rhel
diff --git a/README.md b/README.md
index 999501d..9f4a938 100644
--- a/README.md
+++ b/README.md
@@ -41,45 +41,60 @@ make copy-couch $(lsb_release -cs) COUCHTARBALL=path/to/couchdb-#.#.#.tar.gz PLA
-----
-### Building inside the `couchdbdev` docker containers
+### Building inside the `couchdbci` docker containers
-You must first pull down the image or images you need from Docker Hub, or build the images
-using the [apache/couchdb-ci](https://github.com/apache/couchdb-ci) repository. A full
-list of supported environments is at https://hub.docker.com/u/couchdbdev/ .
+`build.sh` runs the package build as a `docker buildx build` with an
+`apache/couchdbci-<os>:<codename>-erlang-<version>` image from
+DockerHub. Those are built in the
+[apache/couchdb-ci](https://github.com/apache/couchdb-ci) repo. The
+image is pulled automatically and the resulting packages are written
+to `pkgs/couch/<os>-<codename>/`.
-#### SpiderMonkey 1.8.5
+From a downloaded CouchDB tarball:
```shell
-docker pull couchdbdev/<os>-<codename>-base
-./build.sh js <os>-<codename> # for example, debian-stretch, ubuntu-bionic or centos-7.
+ERLANGVERSION=<erlang-version> ./build.sh couch <os>-<codename> path/to/couchdb-#.#.#.tar.gz
```
-#### CouchDB
+Directly from the Apache source CDN:
-From a downloaded CouchDB tarball:
+```shell
+./build.sh couch <os>-<codename> https://dist.apache.org/repos/dist/release/couchdb/source/#.#.#/apache-couchdb-#.#.#.tar.gz
+```
+
+For another architecture:
```shell
-docker pull couchdbdev/<osname>-<codename>-erlang-<erlang-version>
-ERLANGVERSION=<erlang-version> ./build.sh couch <os>-<codename> path/to/couchdb-#.#.#.tar.gz
+CONTAINERARCH=arm64 ./build.sh couch <os>-<codename> path/to/couchdb-#.#.#.tar.gz
```
-Directly from the Apache source CDN:
+The build is run on with buildx. With the `default` builder a foreign
+arch is run under qemu (needs `binfmt_misc` handlers on the host (for
+example `docker run --privileged --rm tonistiigi/binfmt --install
+arm64`). To build natively use an arch specific set of builders. One
+node per arch. For example:
```shell
-docker pull couchdbdev/<osname>-<codename>-erlang-<erlang-version>
-./build.sh couch <os>-<codename> https://dist.apache.org/repos/dist/release/couchdb/source/#.#.#/apache-couchdb-#.#.#.tar.gz
+docker buildx create --name multiarch --use ssh://my-amd64-host
+docker buildx create --name multiarch --append ssh://my-arm64-host
```
+Then, `buildx` will pick the node based on `--platform`. Use
+`BUILDKIT_PROGRESS=auto` for a less noisy buildx progress view.
+
-----
## Building packages for a release
### Prerequisites
-1. Linux running Docker
-1. The current user must be capable of running `docker run`.
-1. Enough free disk space to download all of the Docker images + build
- CouchDB.
+1. Linux running Docker with buildx.
+1. User must be able of running `docker buildx build`.
+1. For non-x86_64 packages, need either qemu `binfmt_misc` handlers,
+ or buildx builders with native nodes (see above).
+1. Enough free disk space on the builder to download all of the Docker
+ images + build CouchDB. Build layers stay in the builder's
+ cache. But can use `docker buildx prune` to clean up if needed.
### Running the package build
@@ -98,18 +113,12 @@ or
Packages will be placed in the `pkgs/couch` subdirectory.
-A similar `js-all` target exists, should the SpiderMonkey packages need to be regenerated.
-
### Uploading the packages
If you have Apache credentials (set your `BINARY_CREDS` environment variable appropriately), after building all CouchDB packages above, **and signing the rpms with the appropriate GPG key using the `rpmsign --addsign <file.rpm>` command**, simply run:
./build.sh couch-upload-all
-Or, for the SpiderMonkey packages:
-
- ./build.sh js-upload-all
-
-----
## Upload Dev Packages
diff --git a/build.sh b/build.sh
index 8501ba0..f3662ac 100755
--- a/build.sh
+++ b/build.sh
@@ -17,12 +17,15 @@
# specific language governing permissions and limitations
# under the License.
-# This is the master shell script to build Docker containers
-# for CouchDB 2.x.
+# This is the master shell script to build CouchDB packages inside the
+# couchdb-ci Docker images.
# stop on error
set -e
+# Full build log. Can set BUILDKIT_PROGRESS=auto for a less noisy log.
+export BUILDKIT_PROGRESS=${BUILDKIT_PROGRESS:-plain}
+
# This works if we're not called through a symlink
# otherwise, see https://stackoverflow.com/questions/59895/
SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@@ -81,26 +84,19 @@ get-couch-tarball() {
}
build-couch() {
+ # $1 - platform, for example: ubuntu-noble
+ # CONTAINERARCH - arch, for example (amd64)
split-os-ver $1
- # We will be changing user to 'jenkins' - ensure it has write permissions
- chmod a+rwx pkgs pkgs/couch pkgs/js
- # $1 is plat, $2 is the optional path to a dist tarball
- if [ -z ${CONTAINERARCH+x} ]; then
- docker run \
- --mount type=bind,src=${SCRIPTPATH},dst=/home/jenkins/couchdb-pkg \
- -u 0 -w /home/jenkins/couchdb-pkg \
- --platform linux/amd64 \
- apache/couchdbci-${os}:${version}-erlang-${ERLANGVERSION} \
- make copy-couch $1 COUCHTARBALL=${COUCHTARBALL}
- else
- docker run \
- --mount type=bind,src=${SCRIPTPATH},dst=/home/jenkins/couchdb-pkg \
- -u 0 -w /home/jenkins/couchdb-pkg \
- --platform linux/${CONTAINERARCH} \
- apache/couchdbci-${os}:${version}-erlang-${ERLANGVERSION} \
- make copy-couch ${CONTAINERARCH}-$1 COUCHTARBALL=${COUCHTARBALL}
- fi
- make clean
+ local arch=${CONTAINERARCH:-amd64}
+ echo " **** building $1 for ${arch} **** "
+ docker buildx build \
+ --platform linux/${arch} \
+ --pull \
+ --build-arg IMAGE=apache/couchdbci-${os}:${version}-erlang-${ERLANGVERSION} \
+ --build-arg TARGET=$1 \
+ --build-arg COUCHTARBALL=${COUCHTARBALL} \
+ --output type=local,dest=${SCRIPTPATH}/pkgs/couch \
+ ${SCRIPTPATH}
}
build-all-couch() {
@@ -119,8 +115,6 @@ build-all-couch() {
( ${base} == "centos-8" && ${arch} == "ppc64le" ) ||
( ${base} == "centos-9" && ${arch} == "ppc64le" ) ||
( ${base} == "centos-10" && ${arch} == "ppc64le") ||
- ( ${base} == "debian-bullseye" && ${arch} == "ppc64le" ) ||
- ( ${base} == "debian-bullseye" && ${arch} == "s390x") ||
( ${base} == "debian-bookworm" && ${arch} == "ppc64le" ) ||
( ${base} == "debian-trixie" && ${arch} == "ppc64le" ) ||
( ${base} == "ubuntu-jammy" && ${arch} == "ppc64le" ) ||
@@ -226,6 +220,9 @@ Recognized commands:
- a URL to http(s)://domain.com/to/couchdb.tar.gz
Commands marked with * require BINARY_CREDS env var.
+
+ Set CONTAINERARCH=arm64 to build a single platform for another arch (arm64).
+ couch-all builds every architecture in XPLAT_ARCHES.
EOF
}