You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2026/10/12 05:14:00 UTC
(couchdb-pkg) branch cleanup-and-buildx-remotes-for-pkgs created (now 1c4560d)
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a change to branch cleanup-and-buildx-remotes-for-pkgs
in repository https://gitbox.apache.org/repos/asf/couchdb-pkg.git
at 1c4560d Cleanup package builds and use remote buildx nodes
This branch includes the following new commits:
new 1c4560d Cleanup package builds and use remote buildx nodes
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(couchdb-pkg) 01/01: Cleanup package builds and use remote buildx nodes
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch cleanup-and-buildx-remotes-for-pkgs
in repository https://gitbox.apache.org/repos/asf/couchdb-pkg.git
commit 1c4560d6ea204de43cc579b4060579b05eac6285
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Mon Oct 12 01:09:52 2026 -0400
Cleanup package builds and use remote buildx nodes
Cleanup some makefile targets (remove the silly redundancy), and also
build package on remote buildx instances, based on architecture/os
combos. That's how CI images are built. And up until now package were
still built on my (host) laptop using emulation which was very slow.
The setup is something like
```
docker buildx rm multiarch || true
docker buildx create --name multiarch --driver docker-container --platform linux/arm64 ssh://ubuntu-nc-arm64-12
docker buildx create --append --name multiarch --driver docker-container --platform linux/amd64 ssh://ubuntu-fsn-1-09
docker buildx create --append --name multiarch --driver docker-container --platform linux/s390x ssh://linux1
docker buildx create --append --name multiarch --driver docker-container --platform linux/ppc64le ssh://ubuntu
```
Re-bootstrap after OS updates
```
docker buildx inspect --bootstrap
```
---
.dockerignore | 14 +++++++
Dockerfile | 36 +++++++++++++++++
Makefile | 121 +++++-----------------------------------------------------
README.md | 57 +++++++++++++++------------
build.sh | 43 ++++++++++-----------
5 files changed, 113 insertions(+), 158 deletions(-)
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..bcdd323
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,14 @@
+# Keep only the package sources and CouchDB tarball
+.git
+.idea
+archive
+pkgs
+pkgs.bak*
+*.bak
+repo/db
+repo/dists
+repo/pool
+parts
+prime
+stage
+*.snap
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..2c4cb4d
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,36 @@
+# check=skip=InvalidDefaultArgInFrom
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+# Build CouchDB packages for an {os,arch} combo inside the matching couchdb-ci image.
+# Run from build.sh as:
+#
+# docker buildx build --platform linux/<arch> --output type=local,dest=pkgs/couch ...
+#
+# The reuslt is the built package(s). Then the local exporter saves
+# them into pkgs/couch/$dist on the host. The build itself run on
+# whatever node of the current builder supports combo (possibly
+# remote).
+
+# IMAGE always passed in by build.sh. No good default for it, so added
+# lint in line 1 so it stops complaining about.
+ARG IMAGE
+FROM ${IMAGE} AS build
+ARG TARGET
+ARG COUCHTARBALL
+USER root
+WORKDIR /home/jenkins/couchdb-pkg
+COPY . .
+RUN make copy-couch ${TARGET} COUCHTARBALL=${COUCHTARBALL}
+
+FROM scratch
+COPY --from=build /home/jenkins/couchdb-pkg/pkgs/couch/ /
diff --git a/Makefile b/Makefile
index 06490fb..ee11b47 100644
--- a/Makefile
+++ b/Makefile
@@ -21,13 +21,9 @@ export DEBEMAIL=dev@couchdb.apache.org
# Default package directory (over-written for RPM based builds)
PKGDIR=$(COUCHDIR)
-ifeq ($(shell arch),aarch64)
-PKGARCH=arm64
-else ifeq ($(shell arch),arm64v8)
-PKGARCH=arm64
-else
+# RPMs go into the RPMS/<arch> dir where where <arch> is `uname -m`
+# (x86_64, aarch64, ppc64le, ...). DEB builds figure out their own arch
PKGARCH=$(shell arch)
-endif
# Overridden for targets with SM60 in per-target env vars below.
SPIDERMONKEY=couch-libmozjs185-1.0
@@ -42,6 +38,14 @@ JAVA_RUNTIME=java21-runtime-headless | java21-runtime
all:
make `bin/detect-target.sh`
+# arch specific prefixes arm64-ubuntu-noble are aliases for plain
+# target
+arm64-%: % ; @:
+arm64v8-%: % ; @:
+aarch64-%: % ; @:
+ppc64le-%: % ; @:
+s390x-%: % ; @:
+
# Debian default
debian: sm-ver-debian find-couch-dist copy-debian update-changelog dpkg lintian copy-pkgs
@@ -53,30 +57,6 @@ debian-bookworm: SPIDERMONKEY_DEV=libmozjs-78-dev
debian-bookworm: SM_VER=78
debian-bookworm: bookworm
-arm64-debian-bookworm: aarch64-debian-bookworm
-arm64v8-debian-bookworm: aarch64-debian-bookworm
-aarch64-debian-bookworm: PLATFORM=bookworm
-aarch64-debian-bookworm: DIST=debian-bookworm
-aarch64-debian-bookworm: SPIDERMONKEY=libmozjs-78-0
-aarch64-debian-bookworm: SPIDERMONKEY_DEV=libmozjs-78-dev
-aarch64-debian-bookworm: SM_VER=78
-aarch64-debian-bookworm: bookworm
-
-ppc64le-debian-bookworm: PLATFORM=bookworm
-ppc64le-debian-bookworm: DIST=debian-bookworm
-ppc64le-debian-bookworm: SPIDERMONKEY=libmozjs-78-0
-ppc64le-debian-bookworm: SPIDERMONKEY_DEV=libmozjs-78-dev
-ppc64le-debian-bookworm: SM_VER=78
-ppc64le-debian-bookworm: bookworm
-
-s390x-debian-bookworm: PLATFORM=bookworm
-s390x-debian-bookworm: DIST=debian-bookworm
-s390x-debian-bookworm: SPIDERMONKEY=libmozjs-78-0
-s390x-debian-bookworm: SPIDERMONKEY_DEV=libmozjs-78-dev
-s390x-debian-bookworm: SM_VER=78
-s390x-debian-bookworm: bookworm
-
-
bookworm: debian
# Debian 13 - trixie
@@ -88,32 +68,6 @@ debian-trixie: SM_VER=128
debian-trixie: JAVA_RUNTIME=java21-runtime-headless | java21-runtime
debian-trixie: trixie
-arm64-debian-trixie: aarch64-debian-trixie
-arm64v8-debian-trixie: aarch64-debian-trixie
-aarch64-debian-trixie: PLATFORM=trixie
-aarch64-debian-trixie: DIST=debian-trixie
-aarch64-debian-trixie: SPIDERMONKEY=libmozjs-128-0
-aarch64-debian-trixie: SPIDERMONKEY_DEV=libmozjs-128-dev
-aarch64-debian-trixie: SM_VER=128
-aarch64-debian-trixie: JAVA_RUNTIME=java21-runtime-headless | java21-runtime
-aarch64-debian-trixie: trixie
-
-ppc64le-debian-trixie: PLATFORM=trixie
-ppc64le-debian-trixie: DIST=debian-trixie
-ppc64le-debian-trixie: SPIDERMONKEY=libmozjs-128-0
-ppc64le-debian-trixie: SPIDERMONKEY_DEV=libmozjs-128-dev
-ppc64le-debian-trixie: SM_VER=128
-ppc64le-debian-trixie: JAVA_RUNTIME=java21-runtime-headless | java21-runtime
-ppc64le-debian-trixie: trixie
-
-s390x-debian-trixie: PLATFORM=trixie
-s390x-debian-trixie: DIST=debian-trixie
-s390x-debian-trixie: SPIDERMONKEY=libmozjs-128-0
-s390x-debian-trixie: SPIDERMONKEY_DEV=libmozjs-128-dev
-s390x-debian-trixie: SM_VER=128
-s390x-debian-trixie: JAVA_RUNTIME=java21-runtime-headless | java21-runtime
-s390x-debian-trixie: trixie
-
trixie: debian
# Ubuntu 22.04 (Jammy)
@@ -125,11 +79,6 @@ ubuntu-jammy: SM_VER=78
ubuntu-jammy: jammy
jammy: debian
-s390x-ubuntu-jammy: ubuntu-jammy
-arm64-ubuntu-jammy: ubuntu-jammy
-aarch64-ubuntu-jammy: ubuntu-jammy
-ppc64le-ubuntu-jammy: ubuntu-jammy
-
# Ubuntu 24.04 (Noble)
ubuntu-noble: PLATFORM=noble
ubuntu-noble: DIST=ubuntu-noble
@@ -139,11 +88,6 @@ ubuntu-noble: SM_VER=115
ubuntu-noble: noble
noble: debian
-s390x-ubuntu-noble: ubuntu-noble
-arm64-ubuntu-noble: ubuntu-noble
-aarch64-ubuntu-noble: ubuntu-noble
-ppc64le-ubuntu-noble: ubuntu-noble
-
# Ubuntu 26.04 (Resolute)
ubuntu-resolute: PLATFORM=resolute
ubuntu-resolute: DIST=ubuntu-resolute
@@ -153,11 +97,6 @@ ubuntu-resolute: SM_VER=128
ubuntu-resolute: resolute
resolute: debian
-s390x-ubuntu-resolute: ubuntu-resolute
-arm64-ubuntu-resolute: ubuntu-resolute
-aarch64-ubuntu-resolute: ubuntu-resolute
-ppc64le-ubuntu-resolute: ubuntu-resolute
-
# RPM default
centos: PKGDIR=../rpmbuild/RPMS/$(PKGARCH)
centos: find-couch-dist link-couch-dist build-rpm copy-pkgs
@@ -186,10 +125,6 @@ almalinux-8: centos-8
almalinux-8.8: centos-8
almalinux-8.9: centos-8
almalinux-8.10: centos-8
-aarch64-almalinux-8.10: PKGARCH=aarch64
-aarch64-almalinux-8.10: centos-8
-aarch64-almalinux-8: PKGARCH=aarch64
-aarch64-almalinux-8: centos-8
# Almalinux 9 is a CentOS 9 alias
almalinux-9: centos-9
@@ -199,47 +134,11 @@ almalinux-9.5: centos-9
almalinux-9.6: centos-9
almalinux-9.7: centos-9
almalinux-9.8: centos-9
-aarch64-almalinux-9.4: PKGARCH=aarch64
-aarch64-almalinux-9.4: centos-9
-aarch64-almalinux-9.5: PKGARCH=aarch64
-aarch64-almalinux-9.5: centos-9
-aarch64-almalinux-9.6: PKGARCH=aarch64
-aarch64-almalinux-9.6: centos-9
-aarch64-almalinux-9.7: PKGARCH=aarch64
-aarch64-almalinux-9.7: centos-9
-aarch64-almalinux-9.8: PKGARCH=aarch64
-aarch64-almalinux-9.8: centos-9
-aarch64-almalinux-9: PKGARCH=aarch64
-aarch64-almalinux-9: centos-9
-# s390x RHEL 8 clone based
-s390x-centos-8: centos-8
-ppc64le-centos-8: centos-8
-# s390x RHEL 9 clone based
-s390x-centos-9: centos-9
-
-arm64-centos-9: PKGARCH=aarch64
-arm64-centos-9: centos-9
-ppc64le-centos-9: centos-9
# Almalinux 10 is a CentOS 10 alias
almalinux-10: centos-10
almalinux-10.1: centos-10
almalinux-10.2: centos-10
-aarch64-almalinux-10: PKGARCH=aarch64
-aarch64-almalinux-10: centos-10
-aarch64-almalinux-10.1: PKGARCH=aarch64
-aarch64-almalinux-10.1: centos-10
-aarch64-almalinux-10.2: PKGARCH=aarch64
-aarch64-almalinux-10.2: centos-10
-# s390x RHEL 10 clone based
-s390x-centos-10: centos-10
-ppc64le-centos-10: centos-10
-# s390x RHEL 10 clone based
-s390x-centos-10: centos-10
-
-arm64-centos-10: PKGARCH=aarch64
-arm64-centos-10: centos-10
-ppc64le-centos-10: centos-10
# aarch64 RHEL-based
aarch64-rhel: DIST=rhel
diff --git a/README.md b/README.md
index 999501d..9f4a938 100644
--- a/README.md
+++ b/README.md
@@ -41,45 +41,60 @@ make copy-couch $(lsb_release -cs) COUCHTARBALL=path/to/couchdb-#.#.#.tar.gz PLA
-----
-### Building inside the `couchdbdev` docker containers
+### Building inside the `couchdbci` docker containers
-You must first pull down the image or images you need from Docker Hub, or build the images
-using the [apache/couchdb-ci](https://github.com/apache/couchdb-ci) repository. A full
-list of supported environments is at https://hub.docker.com/u/couchdbdev/ .
+`build.sh` runs the package build as a `docker buildx build` with an
+`apache/couchdbci-<os>:<codename>-erlang-<version>` image from
+DockerHub. Those are built in the
+[apache/couchdb-ci](https://github.com/apache/couchdb-ci) repo. The
+image is pulled automatically and the resulting packages are written
+to `pkgs/couch/<os>-<codename>/`.
-#### SpiderMonkey 1.8.5
+From a downloaded CouchDB tarball:
```shell
-docker pull couchdbdev/<os>-<codename>-base
-./build.sh js <os>-<codename> # for example, debian-stretch, ubuntu-bionic or centos-7.
+ERLANGVERSION=<erlang-version> ./build.sh couch <os>-<codename> path/to/couchdb-#.#.#.tar.gz
```
-#### CouchDB
+Directly from the Apache source CDN:
-From a downloaded CouchDB tarball:
+```shell
+./build.sh couch <os>-<codename> https://dist.apache.org/repos/dist/release/couchdb/source/#.#.#/apache-couchdb-#.#.#.tar.gz
+```
+
+For another architecture:
```shell
-docker pull couchdbdev/<osname>-<codename>-erlang-<erlang-version>
-ERLANGVERSION=<erlang-version> ./build.sh couch <os>-<codename> path/to/couchdb-#.#.#.tar.gz
+CONTAINERARCH=arm64 ./build.sh couch <os>-<codename> path/to/couchdb-#.#.#.tar.gz
```
-Directly from the Apache source CDN:
+The build is run on with buildx. With the `default` builder a foreign
+arch is run under qemu (needs `binfmt_misc` handlers on the host (for
+example `docker run --privileged --rm tonistiigi/binfmt --install
+arm64`). To build natively use an arch specific set of builders. One
+node per arch. For example:
```shell
-docker pull couchdbdev/<osname>-<codename>-erlang-<erlang-version>
-./build.sh couch <os>-<codename> https://dist.apache.org/repos/dist/release/couchdb/source/#.#.#/apache-couchdb-#.#.#.tar.gz
+docker buildx create --name multiarch --use ssh://my-amd64-host
+docker buildx create --name multiarch --append ssh://my-arm64-host
```
+Then, `buildx` will pick the node based on `--platform`. Use
+`BUILDKIT_PROGRESS=auto` for a less noisy buildx progress view.
+
-----
## Building packages for a release
### Prerequisites
-1. Linux running Docker
-1. The current user must be capable of running `docker run`.
-1. Enough free disk space to download all of the Docker images + build
- CouchDB.
+1. Linux running Docker with buildx.
+1. User must be able of running `docker buildx build`.
+1. For non-x86_64 packages, need either qemu `binfmt_misc` handlers,
+ or buildx builders with native nodes (see above).
+1. Enough free disk space on the builder to download all of the Docker
+ images + build CouchDB. Build layers stay in the builder's
+ cache. But can use `docker buildx prune` to clean up if needed.
### Running the package build
@@ -98,18 +113,12 @@ or
Packages will be placed in the `pkgs/couch` subdirectory.
-A similar `js-all` target exists, should the SpiderMonkey packages need to be regenerated.
-
### Uploading the packages
If you have Apache credentials (set your `BINARY_CREDS` environment variable appropriately), after building all CouchDB packages above, **and signing the rpms with the appropriate GPG key using the `rpmsign --addsign <file.rpm>` command**, simply run:
./build.sh couch-upload-all
-Or, for the SpiderMonkey packages:
-
- ./build.sh js-upload-all
-
-----
## Upload Dev Packages
diff --git a/build.sh b/build.sh
index 8501ba0..f3662ac 100755
--- a/build.sh
+++ b/build.sh
@@ -17,12 +17,15 @@
# specific language governing permissions and limitations
# under the License.
-# This is the master shell script to build Docker containers
-# for CouchDB 2.x.
+# This is the master shell script to build CouchDB packages inside the
+# couchdb-ci Docker images.
# stop on error
set -e
+# Full build log. Can set BUILDKIT_PROGRESS=auto for a less noisy log.
+export BUILDKIT_PROGRESS=${BUILDKIT_PROGRESS:-plain}
+
# This works if we're not called through a symlink
# otherwise, see https://stackoverflow.com/questions/59895/
SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@@ -81,26 +84,19 @@ get-couch-tarball() {
}
build-couch() {
+ # $1 - platform, for example: ubuntu-noble
+ # CONTAINERARCH - arch, for example (amd64)
split-os-ver $1
- # We will be changing user to 'jenkins' - ensure it has write permissions
- chmod a+rwx pkgs pkgs/couch pkgs/js
- # $1 is plat, $2 is the optional path to a dist tarball
- if [ -z ${CONTAINERARCH+x} ]; then
- docker run \
- --mount type=bind,src=${SCRIPTPATH},dst=/home/jenkins/couchdb-pkg \
- -u 0 -w /home/jenkins/couchdb-pkg \
- --platform linux/amd64 \
- apache/couchdbci-${os}:${version}-erlang-${ERLANGVERSION} \
- make copy-couch $1 COUCHTARBALL=${COUCHTARBALL}
- else
- docker run \
- --mount type=bind,src=${SCRIPTPATH},dst=/home/jenkins/couchdb-pkg \
- -u 0 -w /home/jenkins/couchdb-pkg \
- --platform linux/${CONTAINERARCH} \
- apache/couchdbci-${os}:${version}-erlang-${ERLANGVERSION} \
- make copy-couch ${CONTAINERARCH}-$1 COUCHTARBALL=${COUCHTARBALL}
- fi
- make clean
+ local arch=${CONTAINERARCH:-amd64}
+ echo " **** building $1 for ${arch} **** "
+ docker buildx build \
+ --platform linux/${arch} \
+ --pull \
+ --build-arg IMAGE=apache/couchdbci-${os}:${version}-erlang-${ERLANGVERSION} \
+ --build-arg TARGET=$1 \
+ --build-arg COUCHTARBALL=${COUCHTARBALL} \
+ --output type=local,dest=${SCRIPTPATH}/pkgs/couch \
+ ${SCRIPTPATH}
}
build-all-couch() {
@@ -119,8 +115,6 @@ build-all-couch() {
( ${base} == "centos-8" && ${arch} == "ppc64le" ) ||
( ${base} == "centos-9" && ${arch} == "ppc64le" ) ||
( ${base} == "centos-10" && ${arch} == "ppc64le") ||
- ( ${base} == "debian-bullseye" && ${arch} == "ppc64le" ) ||
- ( ${base} == "debian-bullseye" && ${arch} == "s390x") ||
( ${base} == "debian-bookworm" && ${arch} == "ppc64le" ) ||
( ${base} == "debian-trixie" && ${arch} == "ppc64le" ) ||
( ${base} == "ubuntu-jammy" && ${arch} == "ppc64le" ) ||
@@ -226,6 +220,9 @@ Recognized commands:
- a URL to http(s)://domain.com/to/couchdb.tar.gz
Commands marked with * require BINARY_CREDS env var.
+
+ Set CONTAINERARCH=arm64 to build a single platform for another arch (arm64).
+ couch-all builds every architecture in XPLAT_ARCHES.
EOF
}