You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ke...@apache.org on 2025/02/02 20:50:36 UTC
(iceberg-python) branch main updated: automate building Release Candidate in github action (#1391)
This is an automated email from the ASF dual-hosted git repository.
kevinjqliu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new fa782568 automate building Release Candidate in github action (#1391)
fa782568 is described below
commit fa782568eb1792eaaa8bd90674705304d879ba79
Author: Kevin Liu <ke...@users.noreply.github.com>
AuthorDate: Sun Feb 2 15:50:32 2025 -0500
automate building Release Candidate in github action (#1391)
Related to #1306 and #872
Devlist discussion:
https://lists.apache.org/thread/oowhcfwv3fcjzdzm76tbn99k5q84mr75
This PR changes `.github/workflows/python-release.yml` to build the
artifacts for both svn and pypi and update the "how to release doc" to
match the new process.
---
.github/workflows/python-release.yml | 199 +++++++++++++++++++--
mkdocs/docs/api.md | 2 +-
.../images/ghactions-run-workflow-pypi-upload.png | Bin 221264 -> 0 bytes
.../images/ghactions-run-workflow-svn-upload.png | Bin 218863 -> 0 bytes
mkdocs/docs/how-to-release.md | 132 +++++++-------
5 files changed, 253 insertions(+), 80 deletions(-)
diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml
index 7541360b..bba2bdc0 100644
--- a/.github/workflows/python-release.yml
+++ b/.github/workflows/python-release.yml
@@ -17,21 +17,110 @@
# under the License.
#
-name: "Python Release"
+name: "Python Build Release Candidate"
on:
+ push:
+ tags:
+ # Trigger this workflow when tag follows the versioning format: pyiceberg-<major>.<minor>.<patch>rc<release_candidate>
+ # Example valid tags: pyiceberg-0.8.0rc2, pyiceberg-1.0.0rc1
+ - 'pyiceberg-[0-9]+.[0-9]+.[0-9]+rc[0-9]+'
workflow_dispatch:
inputs:
version:
- description: 'Version'
+ description: 'Version (e.g., 0.8.0)'
type: string
- default: 'main'
-
+ required: true
+ rc:
+ description: 'Release Candidate (RC) (e.g., 1)'
+ type: number
+ required: true
jobs:
- build_wheels:
- name: Build wheels on ${{ matrix.os }}
+ validate-inputs:
+ runs-on: ubuntu-latest
+ outputs:
+ VERSION: ${{ steps.validate-inputs.outputs.VERSION }} # e.g. 0.8.0
+ RC: ${{ steps.validate-inputs.outputs.RC }} # e.g. 1
+ steps:
+ - name: Validate and Extract Version and RC
+ id: validate-inputs
+ run: |
+ if [ "$GITHUB_EVENT_NAME" = "push" ]; then
+ echo "Workflow triggered by tag push."
+ TAG=${GITHUB_REF#refs/tags/} # Extract the tag name
+ VERSION_AND_RC=${TAG#pyiceberg-} # Remove the 'pyiceberg-' prefix
+ VERSION=${VERSION_AND_RC%rc*} # Extract VERSION by removing everything after 'rc'
+ RC=${VERSION_AND_RC#*rc} # Extract RC by keeping everything after 'rc'
+
+ if [[ -z "$VERSION" || -z "$RC" ]]; then
+ echo "Error: Unable to parse VERSION or RC from tag ($TAG). Ensure the tag format is correct."
+ exit 1
+ fi
+ else
+ echo "Workflow triggered manually via workflow_dispatch."
+ VERSION="${{ github.event.inputs.version }}"
+ RC="${{ github.event.inputs.rc }}"
+
+ # Validate version (e.g., 1.0.0)
+ if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+ echo "Error: version ($VERSION) must be in the format: <number>.<number>.<number>"
+ exit 1
+ fi
+
+ # Validate rc (e.g., 1)
+ if [[ ! "$RC" =~ ^[0-9]+$ ]]; then
+ echo "Error: rc ($RC) must be in the format: <number>"
+ exit 1
+ fi
+ fi
+
+ # Export variables for future steps
+ echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+ echo "RC=$RC" >> $GITHUB_OUTPUT
+
+ - name: Display Extracted Version and RC
+ run: |
+ echo "Using Version: ${{ steps.validate-inputs.outputs.VERSION }}"
+ echo "Using RC: ${{ steps.validate-inputs.outputs.RC }}"
+
+ validate-library-version:
+ runs-on: ubuntu-latest
+ needs:
+ - validate-inputs
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 1
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: 3.12
+
+ - name: Install Poetry
+ run: make install-poetry
+
+ - name: Validate current pyiceberg version
+ env:
+ VERSION: ${{ needs.validate-inputs.outputs.VERSION }}
+ run: |
+ # Extract the current version from Poetry
+ current_pyiceberg_version=$(poetry version --short)
+ echo "Detected Poetry version: $current_pyiceberg_version"
+
+ # Compare the input version with the Poetry version
+ if [[ "$VERSION" != "$current_pyiceberg_version" ]]; then
+ echo "Error: Input version ($VERSION) does not match the Poetry version ($current_pyiceberg_version)"
+ exit 1
+ fi
+
+ # SVN
+ svn-build-artifacts:
+ name: Build artifacts for SVN on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
+ needs:
+ - validate-inputs
+ - validate-library-version
strategy:
matrix:
os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ]
@@ -39,7 +128,7 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
- fetch-depth: 0
+ fetch-depth: 1
- uses: actions/setup-python@v5
with:
@@ -50,11 +139,86 @@ jobs:
3.12
- name: Install poetry
- run: pip install poetry
+ run: make install-poetry
- - name: Set version
- run: python -m poetry version "${{ inputs.version }}"
- if: "${{ github.event.inputs.version != 'main' }}"
+ # Publish the source distribution with the version that's in
+ # the repository, otherwise the tests will fail
+ - name: Compile source distribution
+ run: python3 -m poetry build --format=sdist
+ if: startsWith(matrix.os, 'ubuntu')
+
+ - name: Build wheels
+ uses: pypa/cibuildwheel@v2.22.0
+ with:
+ output-dir: wheelhouse
+ config-file: "pyproject.toml"
+ env:
+ # Ignore 32 bit architectures
+ CIBW_ARCHS: "auto64"
+ CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13"
+ CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
+ CIBW_TEST_EXTRAS: "s3fs,glue"
+ CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
+ # There is an upstream issue with installing on MacOSX
+ # https://github.com/pypa/cibuildwheel/issues/1603
+ # Ignore tests for pypy since not all dependencies are compiled for it
+ # and would require a local rust build chain
+ CIBW_TEST_SKIP: "pp* *macosx*"
+
+ - name: Add source distribution
+ if: startsWith(matrix.os, 'ubuntu')
+ run: ls -lah dist/* && cp dist/* wheelhouse/
+
+ - uses: actions/upload-artifact@v4
+ with:
+ name: "svn-release-candidate-${{ matrix.os }}"
+ path: ./wheelhouse/*
+
+ svn-merge-artifacts:
+ runs-on: ubuntu-latest
+ needs:
+ - validate-inputs
+ - svn-build-artifacts
+ steps:
+ - name: Merge Artifacts
+ uses: actions/upload-artifact/merge@v4
+ with:
+ name: "svn-release-candidate-${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}"
+ pattern: svn-release-candidate*
+ delete-merged: true
+
+ # PyPi
+ pypi-build-artifacts:
+ name: Build artifacts for PyPi on ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
+ needs:
+ - validate-inputs
+ - validate-library-version
+ strategy:
+ matrix:
+ os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ]
+
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 1
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: |
+ 3.9
+ 3.10
+ 3.11
+ 3.12
+
+ - name: Install poetry
+ run: make install-poetry
+
+ - name: Set version with RC
+ env:
+ VERSION: ${{ needs.validate-inputs.outputs.VERSION }}
+ RC: ${{ needs.validate-inputs.outputs.RC }}
+ run: python -m poetry version "${{ env.VERSION }}rc${{ env.RC }}" # e.g., 0.8.0rc1
# Publish the source distribution with the version that's in
# the repository, otherwise the tests will fail
@@ -86,15 +250,18 @@ jobs:
- uses: actions/upload-artifact@v4
with:
- name: "release-${{ matrix.os }}"
+ name: "pypi-release-candidate-${{ matrix.os }}"
path: ./wheelhouse/*
- merge:
+
+ pypi-merge-artifacts:
runs-on: ubuntu-latest
- needs: build_wheels
+ needs:
+ - validate-inputs
+ - pypi-build-artifacts
steps:
- name: Merge Artifacts
uses: actions/upload-artifact/merge@v4
with:
- name: "release-${{ github.event.inputs.version }}"
- pattern: release-*
+ name: "pypi-release-candidate-${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}"
+ pattern: pypi-release-candidate*
delete-merged: true
diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md
index ce41ec55..b69788ce 100644
--- a/mkdocs/docs/api.md
+++ b/mkdocs/docs/api.md
@@ -1496,7 +1496,7 @@ print(ray_dataset.take(2))
### Daft
-PyIceberg interfaces closely with Daft Dataframes (see also: [Daft integration with Iceberg](https://www.getdaft.io/projects/docs/en/latest/user_guide/integrations/iceberg.html)) which provides a full lazily optimized query engine interface on top of PyIceberg tables.
+PyIceberg interfaces closely with Daft Dataframes (see also: [Daft integration with Iceberg](https://www.getdaft.io/projects/docs/en/stable/integrations/iceberg/)) which provides a full lazily optimized query engine interface on top of PyIceberg tables.
<!-- prettier-ignore-start -->
diff --git a/mkdocs/docs/assets/images/ghactions-run-workflow-pypi-upload.png b/mkdocs/docs/assets/images/ghactions-run-workflow-pypi-upload.png
deleted file mode 100644
index 3e1476b5..00000000
Binary files a/mkdocs/docs/assets/images/ghactions-run-workflow-pypi-upload.png and /dev/null differ
diff --git a/mkdocs/docs/assets/images/ghactions-run-workflow-svn-upload.png b/mkdocs/docs/assets/images/ghactions-run-workflow-svn-upload.png
deleted file mode 100644
index 4577a2dd..00000000
Binary files a/mkdocs/docs/assets/images/ghactions-run-workflow-svn-upload.png and /dev/null differ
diff --git a/mkdocs/docs/how-to-release.md b/mkdocs/docs/how-to-release.md
index c44f56a9..4af3a212 100644
--- a/mkdocs/docs/how-to-release.md
+++ b/mkdocs/docs/how-to-release.md
@@ -109,66 +109,81 @@ Create a signed tag:
Replace `VERSION` and `RC` with the appropriate values for the release.
```bash
-export RC=rc1
-export VERSION=0.7.0${RC}
-export VERSION_WITHOUT_RC=${VERSION/rc?/}
-export VERSION_BRANCH=${VERSION_WITHOUT_RC//./-}
-export GIT_TAG=pyiceberg-${VERSION}
+export VERSION=0.7.0
+export RC=1
-git tag -s ${GIT_TAG} -m "PyIceberg ${VERSION}"
+export VERSION_WITH_RC=${VERSION}rc${RC}
+export GIT_TAG=pyiceberg-${VERSION_WITH_RC}
+
+git tag -s ${GIT_TAG} -m "PyIceberg ${VERSION_WITH_RC}"
git push git@github.com:apache/iceberg-python.git ${GIT_TAG}
```
-### Publish Release Candidate (RC)
+### Create Artifacts
-#### Upload to Apache Dev SVN
+The [`Python Build Release Candidate` Github Action](https://github.com/apache/iceberg-python/actions/workflows/python-release.yml) will run automatically upon tag push.
-##### Create Artifacts for SVN
+This action will generate artifacts that will include both source distribution (`sdist`) and binary distributions (`wheels` using [`cibuildwheel`](https://github.com/pypa/cibuildwheel)) for each architectures.
-Run the [`Python release` Github Action](https://github.com/apache/iceberg-python/actions/workflows/python-release.yml).
+This action will generate two final artifacts:
-* Tag: Use the newly created tag.
-* Version: Set the `version` to `main`, as the source cannot be modified.
+* `svn-release-candidate-${VERSION}rc${RC}` for SVN
+* `pypi-release-candidate-${VERSION}rc${RC}` for PyPi
-
+If `gh` is available, watch the GitHub Action progress using:
+
+```bash
+RUN_ID=$(gh run list --repo apache/iceberg-python --workflow "Python Build Release Candidate" --branch "${GIT_TAG}" --event push --json databaseId -q '.[0].databaseId')
+echo "Waiting for workflow to complete, this will take several minutes..."
+gh run watch $RUN_ID --repo apache/iceberg-python
+```
-This action will generate:
+and download the artifacts using:
+
+```bash
+gh run download $RUN_ID --repo apache/iceberg-python
+```
+
+### Publish Release Candidate (RC)
-* Source distribution (`sdist`)
-* Binary distributions (`wheels`) for each architectures. These are created using [`cibuildwheel`](https://github.com/pypa/cibuildwheel)
+#### Upload to Apache Dev SVN
##### Download Artifacts, Sign, and Generate Checksums
-Download the ZIP file containing the artifacts from the GitHub Actions run and unzip it.
+Download the SVN artifact from the GitHub Action and unzip it.
-Navigate to the release directory. Sign the files and generate checksums:
+Navigate to the artifact directory. Generate signature and checksum files:
* `.asc` files: GPG-signed versions of each artifact to ensure authenticity.
* `.sha512` files: SHA-512 checksums for verifying file integrity.
```bash
-cd release-main/
-
-for name in $(ls pyiceberg-*.whl pyiceberg-*.tar.gz)
-do
- gpg --yes --armor --output "${name}.asc" --detach-sig "${name}"
- shasum -a 512 "${name}" > "${name}.sha512"
-done
+(
+ cd svn-release-candidate-${VERSION}rc${RC}
+
+ for name in $(ls pyiceberg-*.whl pyiceberg-*.tar.gz)
+ do
+ gpg --yes --armor --output "${name}.asc" --detach-sig "${name}"
+ shasum -a 512 "${name}" > "${name}.sha512"
+ done
+)
```
+The parentheses `()` create a subshell. Any changes to the directory (`cd`) are limited to this subshell, so the current directory in the parent shell remains unchanged.
+
##### Upload Artifacts to Apache Dev SVN
Now, upload the files from the same directory:
```bash
-export SVN_TMP_DIR=/tmp/iceberg-${VERSION_BRANCH}/
+export SVN_TMP_DIR=/tmp/iceberg-${VERSION}/
svn checkout https://dist.apache.org/repos/dist/dev/iceberg $SVN_TMP_DIR
-export SVN_TMP_DIR_VERSIONED=${SVN_TMP_DIR}pyiceberg-$VERSION/
+export SVN_TMP_DIR_VERSIONED=${SVN_TMP_DIR}pyiceberg-$VERSION_WITH_RC/
mkdir -p $SVN_TMP_DIR_VERSIONED
-cp * $SVN_TMP_DIR_VERSIONED
+cp svn-release-candidate-${VERSION}rc${RC}/* $SVN_TMP_DIR_VERSIONED
svn add $SVN_TMP_DIR_VERSIONED
-svn ci -m "PyIceberg ${VERSION}" ${SVN_TMP_DIR_VERSIONED}
+svn ci -m "PyIceberg ${VERSION_WITH_RC}" ${SVN_TMP_DIR_VERSIONED}
```
Verify the artifact is uploaded to [https://dist.apache.org/repos/dist/dev/iceberg](https://dist.apache.org/repos/dist/dev/iceberg/).
@@ -183,22 +198,13 @@ svn delete https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-<OLD_RC_VERS
#### Upload to PyPi
-##### Create Artifacts for PyPi
-
-Run the [`Python release` Github Action](https://github.com/apache/iceberg-python/actions/workflows/python-release.yml).
-
-* Tag: Use the newly created tag.
-* Version: Set the `version` to release candidate, e.g. `0.7.0rc1`.
-
-
-
##### Download Artifacts
-Download the zip file from the Github Action run and unzip locally.
+Download the PyPi artifact from the GitHub Action and unzip it.
##### Upload Artifacts to PyPi
-Upload release candidate to PyPi. This **won't** bump the version for everyone that hasn't pinned their version, since it is set to an RC [pre-release and those are ignored](https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#pre-release-versioning).
+Update the artifact directory to PyPi using `twine`. This **won't** bump the version for everyone that hasn't pinned their version, since it is set to an RC [pre-release and those are ignored](https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#pre-release-versioning).
<!-- prettier-ignore-start -->
@@ -208,7 +214,7 @@ Upload release candidate to PyPi. This **won't** bump the version for everyone t
<!-- prettier-ignore-end -->
```bash
-twine upload release-${VERSION}/*
+twine upload pypi-release-candidate-${VERSION}rc${RC}/*
```
Verify the artifact is uploaded to [PyPi](https://pypi.org/project/pyiceberg/#history).
@@ -226,10 +232,10 @@ export LAST_COMMIT_ID=$(git rev-list ${GIT_TAG} 2> /dev/null | head -n 1)
cat << EOF > release-announcement-email.txt
To: dev@iceberg.apache.org
-Subject: [VOTE] Release Apache PyIceberg $VERSION
+Subject: [VOTE] Release Apache PyIceberg $VERSION_WITH_RC
Hi Everyone,
-I propose that we release the following RC as the official PyIceberg $VERSION_WITHOUT_RC release.
+I propose that we release the following RC as the official PyIceberg $VERSION release.
A summary of the high level features:
@@ -243,7 +249,7 @@ The commit ID is $LAST_COMMIT_ID
The release tarball, signature, and checksums are here:
-* https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-$VERSION/
+* https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-$VERSION_WITH_RC/
You can find the KEYS file here:
@@ -251,9 +257,9 @@ You can find the KEYS file here:
Convenience binary artifacts are staged on pypi:
-https://pypi.org/project/pyiceberg/$VERSION/
+https://pypi.org/project/pyiceberg/$VERSION_WITH_RC/
-And can be installed using: pip3 install pyiceberg==$VERSION
+And can be installed using: pip3 install pyiceberg==$VERSION_WITH_RC
Instructions for verifying a release can be found here:
@@ -262,7 +268,7 @@ Instructions for verifying a release can be found here:
Please download, verify, and test.
Please vote in the next 72 hours.
-[ ] +1 Release this as PyIceberg $VERSION_WITHOUT_RC
+[ ] +1 Release this as PyIceberg $VERSION
[ ] +0
[ ] -1 Do not release this because...
EOF
@@ -302,10 +308,10 @@ Kind regards,
<!-- prettier-ignore-end -->
```bash
-export SVN_DEV_DIR_VERSIONED="https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-${VERSION}"
-export SVN_RELEASE_DIR_VERSIONED="https://dist.apache.org/repos/dist/release/iceberg/pyiceberg-${VERSION_WITHOUT_RC}"
+export SVN_DEV_DIR_VERSIONED="https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-${VERSION_WITH_RC}"
+export SVN_RELEASE_DIR_VERSIONED="https://dist.apache.org/repos/dist/release/iceberg/pyiceberg-${VERSION}"
-svn mv ${SVN_DEV_DIR_VERSIONED} ${SVN_RELEASE_DIR_VERSIONED} -m "PyIceberg: Add release ${VERSION_WITHOUT_RC}"
+svn mv ${SVN_DEV_DIR_VERSIONED} ${SVN_RELEASE_DIR_VERSIONED} -m "PyIceberg: Add release ${VERSION}"
```
Verify the artifact is uploaded to [https://dist.apache.org/repos/dist/release/iceberg](https://dist.apache.org/repos/dist/release/iceberg/).
@@ -331,7 +337,7 @@ The latest version can be pushed to PyPi. Check out the Apache SVN and make sure
```bash
svn checkout https://dist.apache.org/repos/dist/release/iceberg /tmp/iceberg-dist-release/
-cd /tmp/iceberg-dist-release/pyiceberg-${VERSION_WITHOUT_RC}
+cd /tmp/iceberg-dist-release/pyiceberg-${VERSION}
twine upload pyiceberg-*.whl pyiceberg-*.tar.gz
```
@@ -359,18 +365,6 @@ This Python release can be downloaded from: https://pypi.org/project/pyiceberg/<
Thanks to everyone for contributing!
```
-### Release the docs
-
-Run the [`Release Docs` Github Action](https://github.com/apache/iceberg-python/actions/workflows/python-release-docs.yml).
-
-### Update the Github template
-
-Make sure to create a PR to update the [GitHub issues template](https://github.com/apache/iceberg-python/blob/main/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml) with the latest version.
-
-### Update the integration tests
-
-Ensure to update the `PYICEBERG_VERSION` in the [Dockerfile](https://github.com/apache/iceberg-python/blob/main/dev/Dockerfile).
-
### Create a Github Release Note
Create a [new Release Note](https://github.com/apache/iceberg-python/releases/new) on the iceberg-python Github repository.
@@ -385,6 +379,18 @@ Then, select the previous release version as the **Previous tag** to use the dif
**Set as the latest release** and **Publish**.
+### Release the docs
+
+Run the [`Release Docs` Github Action](https://github.com/apache/iceberg-python/actions/workflows/python-release-docs.yml).
+
+### Update the Github template
+
+Make sure to create a PR to update the [GitHub issues template](https://github.com/apache/iceberg-python/blob/main/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml) with the latest version.
+
+### Update the integration tests
+
+Ensure to update the `PYICEBERG_VERSION` in the [Dockerfile](https://github.com/apache/iceberg-python/blob/main/dev/Dockerfile).
+
## Misc
### Set up GPG key and Upload to Apache Iceberg KEYS file