You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by wo...@apache.org on 2021/04/21 18:01:32 UTC
[couchdb-ci] branch main updated: bugfixes; bullseye working
This is an automated email from the ASF dual-hosted git repository.
wohali pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb-ci.git
The following commit(s) were added to refs/heads/main by this push:
new f844f56 bugfixes; bullseye working
f844f56 is described below
commit f844f568a3446f50d9a580e712dfd7461ee95621
Author: Joan Touzet <wo...@apache.org>
AuthorDate: Wed Apr 21 11:01:18 2021 -0700
bugfixes; bullseye working
---
README.md | 4 +--
bin/apt-dependencies.sh | 16 +++++++++---
bin/apt-erlang.sh | 4 +--
bin/detect-os.sh | 6 ++++-
bin/source-erlang.sh | 31 +++++++++++++++--------
bin/yum-dependencies.sh | 8 +++---
build.sh | 14 ++++++++---
dockerfiles/debian-bullseye | 61 +++++++++++++++++++++++++++++++++++++++++++++
8 files changed, 118 insertions(+), 26 deletions(-)
diff --git a/README.md b/README.md
index b3deda6..469e6ba 100644
--- a/README.md
+++ b/README.md
@@ -69,10 +69,10 @@ This only works from an `x86_64` build host.
First, configure your machine with the correct dependencies to build multi-arch binaries:
```
-docker run --rm --privileged multiarch/qemu-user-static --reset -p yes --credential yes
+docker run --privileged --rm tonistiigi/binfmt --install all
```
-This is a one-time setup step. The `multiarch/qemu-user-static` docker container run will install the correct qemu static binaries necessary for running foreign architecture binaries on your host machine. It includes special magic to ensure `sudo` works correctly inside a container, too.
+This is a one-time setup step. This docker container run will install the correct qemu static binaries necessary for running foreign architecture binaries on your host machine. It includes special magic to ensure `sudo` works correctly inside a container, too.
Then, override the `CONTAINERARCH` environment variable when starting `build.sh`:
diff --git a/bin/apt-dependencies.sh b/bin/apt-dependencies.sh
index f0a7ebc..d1c058b 100755
--- a/bin/apt-dependencies.sh
+++ b/bin/apt-dependencies.sh
@@ -68,14 +68,19 @@ apt-get install --no-install-recommends -y apt-transport-https curl git pkg-conf
build-essential ca-certificates libcurl4-openssl-dev \
libicu-dev libnspr4-dev \
help2man python3-sphinx \
- curl debhelper devscripts dh-exec dh-python dh-systemd equivs \
+ curl debhelper devscripts dh-exec dh-python equivs \
dialog equivs lintian libwww-perl quilt \
reprepro rsync \
vim-tiny screen procps dirmngr ssh-client
-# python 2 based; gone from focal / bullseye. look for createrepo_c eventually
-# hopefully via: https://github.com/rpm-software-management/createrepo_c/issues/145
-apt-get install --no-install-recommends -y createrepo || true
+# createrepo_c or createrepo, depending on packaging support
+if [ ${VERSION_CODENAME} == "bullseye" ]; then
+ apt-get install --no-install-recommends -y createrepo-c || true
+else
+ # python 2 based; gone from focal / bullseye. look for createrepo_c eventually
+ # hopefully via: https://github.com/rpm-software-management/createrepo_c/issues/145
+ apt-get install --no-install-recommends -y createrepo || true
+fi
if [ ${VERSION_CODENAME} == "xenial" ]; then
apt remove -y python3-venv
@@ -176,6 +181,9 @@ if [ "$1" != "nojs" ]; then
if [ "${VERSION_CODENAME}" == "focal" ]; then
apt-get install --no-install-recommends -y libmozjs-68-dev
fi
+ if [ "${VERSION_CODENAME}" == "bullseye" ]; then
+ apt-get install --no-install-recommends -y libmozjs-78-dev
+ fi
else
# install js build-time dependencies only
# we can't add the CouchDB repo here because the plat may not exist yet
diff --git a/bin/apt-erlang.sh b/bin/apt-erlang.sh
index 81a7c9e..ee1adb0 100755
--- a/bin/apt-erlang.sh
+++ b/bin/apt-erlang.sh
@@ -68,10 +68,10 @@ then
elif [ "${ARCH}" = x86_64 ] && [ "${ERLANGVERSION}" != "all" ]
then
wget https://packages.erlang-solutions.com/erlang-solutions_2.0_all.deb
- dpkg -i erlang-solutions_*_all.deb
+ dpkg -i erlang-solutions_*_all.deb || true
rm erlang-solutions_*_all.deb
# bionic is broken...
- sed -i 's/debian contrib/debian ${VERSION} contrib/' /etc/apt/sources.list.d/erlang-solutions.list
+ sed -i 's/debian contrib/debian ${VERSION} contrib/' /etc/apt/sources.list.d/erlang-solutions.list || true
if [[ ${ERLANGVERSION} == "19.3.6" && ${VERSION} == "bionic" ]]; then
ERLANGVERSION=19.3.6.8
fi
diff --git a/bin/detect-os.sh b/bin/detect-os.sh
index 83ffb51..8359d24 100755
--- a/bin/detect-os.sh
+++ b/bin/detect-os.sh
@@ -82,10 +82,13 @@ case "${OSTYPE}" in
if [[ ${PRETTY_NAME} =~ "Ubuntu" ]]; then
# Ubuntu keeps changing the format of /etc/os-release's VERSION, and
# the codename is buried. Boo. Let's use a fancy regex.
+ VERSION_ID=${VERSION_ID:-$(dpkg --status tzdata|grep Provides|cut -f2 -d'-')}
VERSION_CODENAME=${VERSION_CODENAME:-$(echo ${VERSION} | sed -E 's/([0-9.]+)\W+([A-Za-z\,]+)\W+\(?(\w+)(.*)/\L\3/')}
DISTRIB_CODENAME=${DISTRIB_CODENAME:-${VERSION_CODENAME}}
elif [[ ${PRETTY_NAME} =~ "Debian" ]]; then
- VERSION_CODENAME=${VERSION_CODENAME:-$(echo ${VERSION} | sed -E 's/(.*)\(([^\]+)\)/\2/')}
+ VERSION_ID=${VERSION_ID:-$(dpkg --status tzdata|grep Provides|cut -f2 -d'-')}
+ VERSION=${VERSION:-${VERSION_ID}}
+ VERSION_CODENAME=${VERSION_CODENAME:-$(echo ${VERSION} | sed -E 's/(.*)\(([^\]+)\)/\2/' | sed -E 's/(.*)\/.*/\1/')}
DISTRIB_CODENAME=${DISTRIB_CODENAME:-${VERSION_CODENAME}}
else
echo "Unknown Debian-like OS ${PRETTY_NAME}, aborting..."
@@ -96,6 +99,7 @@ case "${OSTYPE}" in
echo "Detected distribution: ${ID}, version ${VERSION_ID} (${VERSION_CODENAME})"
else
echo "Unable to determine Linux distribution! Aborting."
+ echo "Detected: ID=${ID}, VERSION_ID=${VERSION_ID}, VERSION_CODENAME=(${VERSION_CODENAME})"
exit 1
fi
;;
diff --git a/bin/source-erlang.sh b/bin/source-erlang.sh
index f5ada8d..2c24386 100755
--- a/bin/source-erlang.sh
+++ b/bin/source-erlang.sh
@@ -44,21 +44,29 @@ latest='(stretch|buster|bionic)'
echo "Erlang source build started @ $(date)"
+export CC=gcc
+export CXX=gcc
+
# Install per-distro dependencies according to:
# http://docs.basho.com/riak/1.3.0/tutorials/installation/Installing-Erlang/
# NB: Dropping suggested superfluous packages; fop and unixodbc-dev
if [[ ${ID} =~ ${redhats} ]]; then
- yum install -y git gcc glibc-devel make ncurses-devel openssl-devel autoconf procps
+ yum install -y git gcc glibc-devel make ncurses-devel openssl-devel autoconf procps
elif [[ ${ID} =~ ${debians} ]]; then
- if [[ ${ERLANGVERSION%%.*} -le 19 ]] && [[ ${VERSION_CODENAME} =~ ${latest} ]] && [ ${ERLANGVERSION} != "all" ]; then
- echo "Recent versions of Linux (Stretch, Bionic, etc) provide a version of libssl"
- echo "which is too new to complile earlier (<=19) versions of Erlang. Please"
- echo "either choose an earlier distro release or a more rencent version of Erlang."
- exit 1
- fi
-
- apt-get update
- apt-get install -y git build-essential autoconf libncurses5-dev openssl libssl-dev xsltproc procps
+ if [[ ${ERLANGVERSION%%.*} -le 19 ]] && [[ ${VERSION_CODENAME} =~ ${latest} ]] && [ ${ERLANGVERSION} != "all" ]; then
+ echo "Recent versions of Linux (Stretch, Bionic, etc) provide a version of libssl"
+ echo "which is too new to complile earlier (<=19) versions of Erlang. Please"
+ echo "either choose an earlier distro release or a more rencent version of Erlang."
+ exit 1
+ fi
+
+ apt-get update
+ apt-get install -y git build-essential autoconf libncurses5-dev openssl libssl-dev xsltproc procps
+ if [ ${VERSION_CODENAME} == "bullseye" ]; then
+ apt install -y gcc-9
+ export CC=gcc-9
+ export CXX=g++-9
+ fi
else
echo "Sorry, we don't support this Linux (${ID}) yet."
exit 1
@@ -118,4 +126,7 @@ elif [[ ${ID} =~ ${debians} ]]; then
apt-get clean
fi
+unset CC
+unset CXX
+
echo "Erlang source build finished @ $(date)"
diff --git a/bin/yum-dependencies.sh b/bin/yum-dependencies.sh
index 9174c03..64d08ee 100755
--- a/bin/yum-dependencies.sh
+++ b/bin/yum-dependencies.sh
@@ -200,12 +200,12 @@ fi
# FoundationDB
if [[ ${VERSION_ID} -eq 6 ]]; then
- wget https://www.foundationdb.org/downloads/6.3.9/rhel6/installers/foundationdb-clients-6.3.29-1.el6.x86_64.rpm
- wget https://www.foundationdb.org/downloads/6.3.9/rhel6/installers/foundationdb-server-6.3.29-1.el6.x86_64.rpm
+ wget https://www.foundationdb.org/downloads/6.3.9/rhel6/installers/foundationdb-clients-6.3.9-1.el6.x86_64.rpm
+ wget https://www.foundationdb.org/downloads/6.3.9/rhel6/installers/foundationdb-server-6.3.9-1.el6.x86_64.rpm
yum --nogpgcheck localinstall -y foundationdb*rpm
else
- wget https://www.foundationdb.org/downloads/6.3.9/rhel7/installers/foundationdb-clients-6.3.29-1.el7.x86_64.rpm
- wget https://www.foundationdb.org/downloads/6.3.9/rhel7/installers/foundationdb-server-6.3.29-1.el7.x86_64.rpm
+ wget https://www.foundationdb.org/downloads/6.3.9/rhel7/installers/foundationdb-clients-6.3.9-1.el7.x86_64.rpm
+ wget https://www.foundationdb.org/downloads/6.3.9/rhel7/installers/foundationdb-server-6.3.9-1.el7.x86_64.rpm
# Buggy FoundationDB packages require this workaround
rpm -i --nodeps ./foundationdb*rpm
fi
diff --git a/build.sh b/build.sh
index 083b09f..23749f4 100755
--- a/build.sh
+++ b/build.sh
@@ -42,7 +42,7 @@ SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# https://ubuntu.com/about/release-cycle
# https://access.redhat.com/support/policy/updates/errata/ (same for CentOS)
# also https://endoflife.software/operating-systems/linux/centos
-DEBIANS="debian-stretch debian-buster"
+DEBIANS="debian-stretch debian-buster debian-bullseye"
UBUNTUS="ubuntu-xenial ubuntu-bionic ubuntu-focal"
CENTOSES="centos-7 centos-8"
ERLANGALL_BASE="debian-buster"
@@ -144,8 +144,9 @@ upload-platform() {
exit 1
fi
find-erlang-version $1
+ check-envs
split-os-ver $1
- docker push apache/couchdbci-${os}:${version}-erlang-${ERLANGVERSION}
+ docker push apache/couchdbci-${os}:${CONTAINERARCH}${version}-erlang-${ERLANGVERSION}
}
build-test-couch() {
@@ -199,6 +200,13 @@ case "$1" in
shift
build-platform $1
;;
+ platform-foreign)
+ # makes only foreign arch platforms
+ shift
+ for arch in $XPLAT_ARCHES; do
+ CONTAINERARCH=$arch build-platform $XPLAT_BASE
+ done
+ ;;
platform-all)
# build all platforms with JS and Erlang support
shift
@@ -220,7 +228,7 @@ case "$1" in
upload-platform $plat $*
done
for arch in $XPLAT_ARCHES; do
- CONTAINERARCH=$arch upload-platform $arch-$XPLAT_BASE $*
+ CONTAINERARCH=$arch upload-platform $XPLAT_BASE $*
done
ERLANGVERSION=all upload-platform $ERLANGALL_BASE
;;
diff --git a/dockerfiles/debian-bullseye b/dockerfiles/debian-bullseye
new file mode 100644
index 0000000..5a109d7
--- /dev/null
+++ b/dockerfiles/debian-bullseye
@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+
+# NOTE: These are intended to be built using the arguments as
+# described in ../build.sh. See that script for more details.
+
+# Support multi-platform builds with a single Dockerfile
+ARG containerarch=amd64
+
+FROM $containerarch/debian:bullseye
+
+# Choose whether to install SpiderMonkey 1.8.5, default yes
+ARG js=js
+# Choose whether to install Erlang, default yes
+ARG erlang=erlang
+# Select version of Node, Erlang and Elixir
+ARG erlangversion=20.3.8.26-1
+ARG kerlversions=20.3.8.26,21.3.8.22,22.3.4.17,23.3.1
+ARG elixirversion=v1.9.4
+ARG nodeversion=14
+
+# Create Jenkins user and group
+RUN groupadd --gid 910 jenkins; \
+ useradd --uid 910 --gid jenkins --create-home jenkins
+
+# Copy couchdb-ci repo into root's home directory
+ADD --chown=root:root bin /root/couchdb-ci/bin/
+ADD --chown=root:root files /root/couchdb-ci/files/
+
+# Jenkins builds in /usr/src/couchdb.
+RUN mkdir -p /usr/src/couchdb; \
+ chown -R jenkins:jenkins /usr/src/couchdb
+
+# Install all dependencies, and optionally SM 1.8.5
+# This allows us to use the same Dockerfile for building SM
+RUN ERLANGVERSION=$erlangversion \
+ KERLVERSIONS=$kerlversions \
+ ELIXIRVERSION=$elixirversion \
+ NODEVERSION=$nodeversion \
+ /root/couchdb-ci/bin/install-dependencies.sh $js $erlang
+
+# Allow Jenkins to sudo
+RUN echo "jenkins ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/jenkins
+
+USER jenkins
+
+# overwrite this with 'CMD []' in a dependent Dockerfile
+CMD ["/bin/bash"]