You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2022/08/17 08:08:29 UTC
[buildstream] 03/06: Build and release wheels packages as part of GitHub Actions
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/build-wheels
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 163c65370aff5b346f092d69f58efa166b1c76e6
Author: Sam Thursfield <sa...@codethink.co.uk>
AuthorDate: Tue Aug 16 18:04:31 2022 +0200
Build and release wheels packages as part of GitHub Actions
This updates the CI config to:
* build wheel packages in pre-merge CI
* download BuildBox release binaries to build the wheels
* these can be downloaded as action artifacts
* test built wheel packages in pre-merge CI
* build sdist and wheel packages on each release tag, and upload
them to Test PyPI at https://test.pypi.org/project/BuildStream/
The new workflows are based on examples at
https://cibuildwheel.readthedocs.io/en/stable/setup/, avoiding use of
the GitHub Actions from `pypa/*` as these are not permitted to be used
in apache/buildstream project.
---
.github/compose/ci.docker-compose.yml | 24 +++++
.../wheel-helpers/fetch-latest-buildbox-release.sh | 16 +++
.github/wheel-helpers/test-wheel-manylinux.sh | 20 ++++
.github/workflows/ci.yml | 54 +++++++++-
.github/workflows/merge.yml | 2 +-
.github/workflows/release.yml | 111 ++++++++++++++++++++-
6 files changed, 222 insertions(+), 5 deletions(-)
diff --git a/.github/compose/ci.docker-compose.yml b/.github/compose/ci.docker-compose.yml
index 83677af1e..f616f1a7b 100644
--- a/.github/compose/ci.docker-compose.yml
+++ b/.github/compose/ci.docker-compose.yml
@@ -85,3 +85,27 @@ services:
mypy:
<<: *tests-template
command: tox -e mypy
+
+ # Test that each BuildStream+BuildBox wheel package can install and run.
+ # on the PyPA official 'manylinux' images that define the base ABI for
+ # Python binary packages.
+
+ wheels-manylinux_2_28-cp37:
+ <<: *tests-template
+ image: quay.io/pypa/manylinux_2_28_x86_64
+ command: .github/wheel-helpers/test-wheel-manylinux.sh cp37-cp37m-manylinux_2_28_x86_64 /opt/python/cp37-cp37m/bin/python3
+
+ wheels-manylinux_2_28-cp38:
+ <<: *tests-template
+ image: quay.io/pypa/manylinux_2_28_x86_64
+ command: .github/wheel-helpers/test-wheel-manylinux.sh cp38-cp38-manylinux_2_28_x86_64 /opt/python/cp38-cp38/bin/python3
+
+ wheels-manylinux_2_28-cp39:
+ <<: *tests-template
+ image: quay.io/pypa/manylinux_2_28_x86_64
+ command: .github/wheel-helpers/test-wheel-manylinux.sh cp39-cp39-manylinux_2_28_x86_64 /opt/python/cp39-cp39/bin/python3
+
+ wheels-manylinux_2_28-cp310:
+ <<: *tests-template
+ image: quay.io/pypa/manylinux_2_28_x86_64
+ command: .github/wheel-helpers/test-wheel-manylinux.sh cp310-cp310-manylinux_2_28_x86_64 /opt/python/cp310-cp310/bin/python3
diff --git a/.github/wheel-helpers/fetch-latest-buildbox-release.sh b/.github/wheel-helpers/fetch-latest-buildbox-release.sh
new file mode 100755
index 000000000..e028ce79f
--- /dev/null
+++ b/.github/wheel-helpers/fetch-latest-buildbox-release.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Download latest release binaries of BuildBox. These are statically linked
+# binaries produced by the buildbox-integration GitLab project, which we
+# bundle into BuildStream wheel packages.
+
+set -eux
+
+wget https://gitlab.com/buildgrid/buildbox/buildbox-integration/-/releases/permalink/latest/downloads/binaries.tgz
+
+mkdir -p src/buildstream/subprojects/buildbox
+tar --extract --file ./binaries.tgz --directory src/buildstream/subprojects/buildbox
+
+cd src/buildstream/subprojects/buildbox
+rm buildbox-run
+mv buildbox-run-bubblewrap buildbox-run
diff --git a/.github/wheel-helpers/test-wheel-manylinux.sh b/.github/wheel-helpers/test-wheel-manylinux.sh
new file mode 100755
index 000000000..a780ab412
--- /dev/null
+++ b/.github/wheel-helpers/test-wheel-manylinux.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# Script to test that a generated BuildStream+BuildBox wheel package is
+# functional in the PyPA "manylinux" container images.
+#
+# The test is run via `run-ci.sh` which in turn uses `docker-compose` to
+# execute this script.
+
+set -eux
+
+COMPATIBILITY_TAGS=$1
+PYTHON=$2
+
+dnf install -y bubblewrap
+
+"$PYTHON" -m venv /tmp/venv
+/tmp/venv/bin/pip3 install ./wheelhouse/BuildStream-*-$COMPATIBILITY_TAGS.whl buildstream-plugins
+
+cd doc/examples/autotools
+/tmp/venv/bin/bst build hello.bst
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c3b3e4bfc..81d143576 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -83,7 +83,7 @@ jobs:
run: |
${GITHUB_WORKSPACE}/.github/run-ci.sh --service ${{ matrix.test-name }}
- docs:
+ build_docs:
runs-on: ubuntu-18.04
steps:
- name: Check out repository
@@ -108,3 +108,55 @@ jobs:
with:
name: docs
path: doc/build/html
+
+ build_wheels:
+ name: Build Python wheel packages on ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [ubuntu-20.04]
+
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - name: Fetch latest BuildBox release
+ run: ${GITHUB_WORKSPACE}/.github/wheel-helpers/fetch-latest-buildbox-release.sh
+
+ - name: Build wheels
+ run: pipx run cibuildwheel==2.8.1
+
+ - uses: actions/upload-artifact@v3
+ with:
+ name: wheels
+ path: ./wheelhouse/*.whl
+
+ test_wheels:
+ name: "Test Python packages: ${{ matrix.test-name }}"
+ needs: [build_wheels]
+ runs-on: ubuntu-20.04
+
+ strategy:
+ matrix:
+ # The names here should map to a valid service defined in
+ # "../compose/ci.docker-compose.yml"
+ test-name:
+ - wheels-manylinux_2_28-cp37
+ - wheels-manylinux_2_28-cp38
+ - wheels-manylinux_2_28-cp39
+ - wheels-manylinux_2_28-cp310
+
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - uses: actions/download-artifact@v3
+ with:
+ name: wheels
+ path: ./wheelhouse
+
+ - name: Run tests with Docker Compose
+ run: |
+ ${GITHUB_WORKSPACE}/.github/run-ci.sh ${{ matrix.test-name }}
diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml
index b9c526375..d9bf499e2 100644
--- a/.github/workflows/merge.yml
+++ b/.github/workflows/merge.yml
@@ -7,7 +7,7 @@ on:
jobs:
build_docs:
- name: Build documentation
+ name: "Build documentation tarball"
runs-on: ubuntu-18.04
steps:
- name: Checkout code
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 36f54c3ed..ccd0f717c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,4 +1,4 @@
-name: Upload Release Asset
+name: Release actions
on:
push:
@@ -6,8 +6,8 @@ on:
- '*.*.*'
jobs:
- build:
- name: Upload Release Asset
+ build_docs:
+ name: "Build documentation tarball"
runs-on: ubuntu-18.04
steps:
- name: Checkout code
@@ -32,9 +32,114 @@ jobs:
tar -C doc/build/html -zcf docs.tgz .
+ - uses: actions/upload-artifact@v3
+ with:
+ name: docs
+ path: docs.tgz
+
+ build_sdist:
+ name: "Build Python source distribution tarball"
+ runs-on: ubuntu-20.04
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - name: Build sdist
+ run: pipx run build --sdist
+
+ - uses: actions/upload-artifact@v3
+ with:
+ name: sdist
+ path: dist/*.tar.gz
+
+ build_wheels:
+ name: Build Python wheel packages on ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [ubuntu-20.04]
+
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - name: Fetch latest BuildBox release
+ run: ${GITHUB_WORKSPACE}/.github/wheel-helpers/fetch-latest-buildbox-release.sh
+
+ - name: Build wheels
+ run: pipx run cibuildwheel==2.8.1
+
+ - uses: actions/upload-artifact@v3
+ with:
+ name: wheels
+ path: ./wheelhouse/*.whl
+
+ test_wheels:
+ name: Test Python wheel packages on ${{ matrix.os }}
+ needs: [build_wheels]
+ runs-on: ubuntu-20.04
+
+ strategy:
+ matrix:
+ # The names here should map to a valid service defined in
+ # "../compose/ci.docker-compose.yml"
+ test-name:
+ - wheels-manylinux_2_28-cp37
+ - wheels-manylinux_2_28-cp38
+ - wheels-manylinux_2_28-cp39
+ - wheels-manylinux_2_28-cp310
+
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - uses: actions/download-artifact@v3
+ with:
+ name: wheels
+ path: ./wheelhouse
+
+ - name: Run tests with Docker Compose
+ run: |
+ ${GITHUB_WORKSPACE}/.github/run-ci.sh ${{ matrix.test-name }}
+
+ upload_github_release:
+ name: Upload GitHub release assets
+ needs: [build_docs, build_sdist, build_wheels, test_wheels]
+ runs-on: ubuntu-20.04
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - uses: actions/download-artifact@v3
+ with:
+ name: docs
+
- name: Upload release assets
run: |
tag_name="${GITHUB_REF##*/}"
hub release create -a "docs.tgz" -m "$tag_name" "$tag_name"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ upload_pypi_release:
+ name: Upload PyPI release assets
+ needs: [build_docs, build_sdist, build_wheels, test_wheels]
+ runs-on: ubuntu-20.04
+ steps:
+ - uses: actions/download-artifact@v3
+ with:
+ name: sdist
+ path: dist
+
+ - uses: actions/download-artifact@v3
+ with:
+ name: wheels
+ path: dist
+
+ - name: Upload to PyPI
+ run: |
+ pipx run twine upload --repository pypi --username __token__ --password "${{ secrets.PYPI_TOKEN }}" dist/*