You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by ad...@apache.org on 2026/04/28 10:08:26 UTC
(polaris) branch main updated: Add PyPI Publishing to Automated Release (#4220)
This is an automated email from the ASF dual-hosted git repository.
adnanhemani pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 88b416bd23 Add PyPI Publishing to Automated Release (#4220)
88b416bd23 is described below
commit 88b416bd2340457f1cadbfc6685d5706322b5dc0
Author: Adnan Hemani <ad...@berkeley.edu>
AuthorDate: Tue Apr 28 03:08:21 2026 -0700
Add PyPI Publishing to Automated Release (#4220)
* Add PyPI Publishing to Automated Release
* Add PyPI Publishing to Automated Release
* remove fork guard
* fix spurrious change
* fix up for PR
* refactor
* fork changes
* zizmor and bug fix
* updating to allowlisted action version
* revert testing changes
* review comments
* Update Makefile
Co-authored-by: Kevin Liu <ke...@users.noreply.github.com>
* Upload Python Wheel file to SVN, Use SVN to upload to PyPI
* ensure both wheel and source files are uploaded
* resolve merge conflict
---------
Co-authored-by: Kevin Liu <ke...@users.noreply.github.com>
---
.github/workflows/nightly.yml | 16 ++-
.../release-3-build-and-publish-artifacts.yml | 127 ++++++++++++++++++++-
.github/workflows/release-4-publish-release.yml | 96 +++++++++++++++-
Makefile | 42 ++++---
client/python/pyproject.toml | 4 +-
releasey/libs/_exec.sh | 12 +-
6 files changed, 269 insertions(+), 28 deletions(-)
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 1a17b99e6a..613f7ada1f 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -76,6 +76,9 @@ jobs:
nightly_build_python_client:
runs-on: ubuntu-latest
if: github.repository == 'apache/polaris'
+ permissions:
+ # IMPORTANT: this permission is mandatory for Trusted Publishing to PyPI
+ id-token: write
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
@@ -93,8 +96,13 @@ jobs:
distribution: 'temurin'
java-version: '21'
- - name: Publish Python client to Test PyPI
- env:
- UV_PUBLISH_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} # zizmor: ignore[secrets-outside-env]
+ - name: Build Python client nightly
run: |
- make client-nightly-publish
+ make client-nightly-build
+
+ - name: Publish Python client to Test PyPI
+ uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
+ with:
+ repository-url: https://test.pypi.org/legacy/
+ packages-dir: client/python/dist/
+ skip-existing: true
diff --git a/.github/workflows/release-3-build-and-publish-artifacts.yml b/.github/workflows/release-3-build-and-publish-artifacts.yml
index a7eb7fa275..b555fbe7b1 100644
--- a/.github/workflows/release-3-build-and-publish-artifacts.yml
+++ b/.github/workflows/release-3-build-and-publish-artifacts.yml
@@ -523,10 +523,129 @@ jobs:
| Artifact Hub metadata | ✅ Committed to dist dev |
EOT
+ publish-python-client-rc:
+ name: Publish Python Client RC to PyPI
+ runs-on: ubuntu-latest
+ needs: [prerequisite-checks]
+ permissions:
+ contents: read
+ # IMPORTANT: this permission is mandatory for Trusted Publishing to PyPI
+ id-token: write
+ env:
+ DRY_RUN: ${{ needs.prerequisite-checks.outputs.dry_run }}
+ version_without_rc: ${{ needs.prerequisite-checks.outputs.version_without_rc }}
+ rc_number: ${{ needs.prerequisite-checks.outputs.rc_number }}
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Set up environment variables
+ run: |
+ echo "RELEASEY_DIR=$(pwd)/releasey" >> $GITHUB_ENV
+ echo "LIBS_DIR=$(pwd)/releasey/libs" >> $GITHUB_ENV
+
+ - name: Set up Python
+ uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
+ with:
+ python-version: "3.13"
+
+ - name: Set up JDK for openapi-generator-cli
+ uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
+ with:
+ distribution: 'temurin'
+ java-version: '21'
+
+ - name: Install Subversion
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y subversion
+
+ - name: Import GPG key
+ uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7.0.0
+ with:
+ gpg_private_key: ${{ secrets.POLARIS_GPG_PRIVATE_KEY }} # zizmor: ignore[secrets-outside-env]
+ git_user_signingkey: true
+ git_commit_gpgsign: true
+
+ - name: Build Python client RC for SVN
+ run: |
+ make client-rc-build RC_VERSION="${version_without_rc}" RC_NUMBER="${rc_number}"
+
+ - name: Sign and checksum Python wheel
+ run: |
+ source "${LIBS_DIR}/_exec.sh"
+
+ wheel_file=$(ls client/python/dist/*.whl)
+ calculate_sha512 "${wheel_file}"
+ exec_process gpg --armor --output "${wheel_file}.asc" --detach-sig "${wheel_file}"
+
+ - name: Stage Python wheel to Apache dist dev repository
+ env:
+ SVN_USERNAME: ${{ secrets.POLARIS_SVN_DEV_USERNAME }} # zizmor: ignore[secrets-outside-env]
+ SVN_PASSWORD: ${{ secrets.POLARIS_SVN_DEV_PASSWORD }} # zizmor: ignore[secrets-outside-env]
+ run: |
+ echo "::add-mask::$SVN_PASSWORD"
+
+ source "${LIBS_DIR}/_constants.sh"
+ source "${LIBS_DIR}/_exec.sh"
+
+ dist_dev_dir=${RELEASEY_DIR}/polaris-dist-dev
+
+ # Retry logic for SVN checkout (Apache SVN can have transient connectivity issues)
+ exec_process_with_retries 5 60 "${dist_dev_dir}" svn checkout --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive "${APACHE_DIST_URL}${APACHE_DIST_PATH}" "${dist_dev_dir}"
+
+ python_client_version_dir="${dist_dev_dir}/python-client/${version_without_rc}"
+
+ # If the python-client version directory already exists in SVN (e.g. re-running the workflow),
+ # delete it so we can re-add fresh artifacts without "already versioned" errors.
+ if svn info "${python_client_version_dir}" &>/dev/null 2>&1; then
+ exec_process svn delete "${python_client_version_dir}"
+ fi
+
+ exec_process mkdir -p "${python_client_version_dir}"
+ exec_process cp client/python/dist/* "${python_client_version_dir}/"
+
+ exec_process cd "${dist_dev_dir}"
+ exec_process svn add --parents "python-client/${version_without_rc}"
+
+ exec_process svn commit --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive -m "Stage Apache Polaris Python client ${version_without_rc} RC${rc_number}"
+
+ - name: Build Python client RC for Test PyPI
+ run: |
+ make client-rc-testpypi-build RC_VERSION="${version_without_rc}" RC_NUMBER="${rc_number}"
+
+ - name: Publish Python client RC to Test PyPI
+ uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
+ with:
+ repository-url: https://test.pypi.org/legacy/
+ packages-dir: client/python/dist/
+
+ - name: Update step summary
+ run: |
+ echo "## Python Client RC" >> $GITHUB_STEP_SUMMARY
+ cat <<EOT >> $GITHUB_STEP_SUMMARY
+ 🎉 Python client RC published successfully:
+
+ | Property | Value |
+ | --- | --- |
+ | Version | \`${version_without_rc}\` |
+ | Apache dist dev | \`python-client/${version_without_rc}\` |
+ | Registry | Test PyPI |
+ EOT
+
generate-release-email:
name: Generate Release Email Body
runs-on: ubuntu-latest
- needs: [prerequisite-checks, build-and-publish-artifacts, build-docker, build-and-stage-helm-chart]
+ needs:
+ - prerequisite-checks
+ - build-and-publish-artifacts
+ - build-docker
+ - build-and-stage-helm-chart
+ - publish-python-client-rc
permissions:
contents: read
env:
@@ -581,6 +700,12 @@ jobs:
NB: you have to build the Docker images locally in order to test Helm charts.
+ The Python CLI wheel is available on:
+ * https://dist.apache.org/repos/dist/dev/polaris/python-client/${version_without_rc}
+
+ The Python CLI is also available on Test PyPI:
+ * https://test.pypi.org/project/apache-polaris/${version_without_rc}rc${rc_number}/
+
You can find the KEYS file here:
* https://downloads.apache.org/polaris/KEYS
diff --git a/.github/workflows/release-4-publish-release.yml b/.github/workflows/release-4-publish-release.yml
index cd6c545b59..c048b2eedc 100644
--- a/.github/workflows/release-4-publish-release.yml
+++ b/.github/workflows/release-4-publish-release.yml
@@ -38,6 +38,8 @@ jobs:
runs-on: ubuntu-latest
permissions:
contents: write
+ outputs:
+ version_without_rc: ${{ steps.release-params.outputs.version_without_rc }}
steps:
- name: Checkout repository
@@ -87,6 +89,7 @@ jobs:
sudo apt-get install -y subversion
- name: Auto-determine release parameters from branch and Git state
+ id: release-params
env:
GIT_REF: ${{ github.ref }}
run: |
@@ -168,6 +171,7 @@ jobs:
# Export variables for next steps
echo "version_without_rc=${version_without_rc}" >> $GITHUB_ENV
+ echo "version_without_rc=${version_without_rc}" >> $GITHUB_OUTPUT
echo "rc_tag=${rc_tag}" >> $GITHUB_ENV
echo "final_release_tag=${final_release_tag}" >> $GITHUB_ENV
echo "release_branch=${current_branch}" >> $GITHUB_ENV
@@ -199,6 +203,9 @@ jobs:
dev_helm_url="${APACHE_DIST_URL}/dev/polaris/helm-chart/${version_without_rc}"
release_helm_url="${APACHE_DIST_URL}/release/polaris/helm-chart/${version_without_rc}"
+ dev_python_client_url="${APACHE_DIST_URL}/dev/polaris/python-client/${version_without_rc}"
+ release_python_client_url="${APACHE_DIST_URL}/release/polaris/python-client/${version_without_rc}"
+
exec_process svn mv --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \
"${dev_artifacts_url}" "${release_artifacts_url}" \
-m "Release Apache Polaris ${version_without_rc}"
@@ -207,9 +214,13 @@ jobs:
"${dev_helm_url}" "${release_helm_url}" \
-m "Release Apache Polaris Helm chart ${version_without_rc}"
+ exec_process svn mv --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \
+ "${dev_python_client_url}" "${release_python_client_url}" \
+ -m "Release Apache Polaris Python client ${version_without_rc}"
+
cat <<EOT >> $GITHUB_STEP_SUMMARY
## Distribution
- Artifacts and Helm chart moved from dist dev to dist release
+ Artifacts, Helm chart, and Python client moved from dist dev to dist release
EOT
- name: Clean up old releases from dist release repository
@@ -265,6 +276,27 @@ jobs:
echo "No old Helm chart versions found to remove" >> $GITHUB_STEP_SUMMARY
fi
+ # List and remove old Python client versions
+ release_python_client_url="${APACHE_DIST_URL}/release/polaris/python-client"
+ old_python_client_versions=$(svn list --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive "${release_python_client_url}" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+(-incubating)?/$' | sed 's|/$||' | grep -v "^${version_without_rc}$" || true)
+
+ if [[ -n "${old_python_client_versions}" ]]; then
+ echo "" >> $GITHUB_STEP_SUMMARY
+ echo "Removing old Python client versions:" >> $GITHUB_STEP_SUMMARY
+ echo '```' >> $GITHUB_STEP_SUMMARY
+ echo "${old_python_client_versions}" >> $GITHUB_STEP_SUMMARY
+ echo '```' >> $GITHUB_STEP_SUMMARY
+
+ for old_python_client_version in ${old_python_client_versions}; do
+ exec_process svn rm --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \
+ "${release_python_client_url}/${old_python_client_version}" \
+ -m "Remove old Python client ${old_python_client_version} (superseded by ${version_without_rc})"
+ done
+ else
+ echo "" >> $GITHUB_STEP_SUMMARY
+ echo "No old Python client versions found to remove" >> $GITHUB_STEP_SUMMARY
+ fi
+
- name: Transfer Helm index and artifacthub-repo.yml from dev to release
env:
SVN_USERNAME: ${{ secrets.POLARIS_SVN_DEV_USERNAME }} # zizmor: ignore[secrets-outside-env]
@@ -446,6 +478,13 @@ jobs:
# Use the Gradle task to release the Apache staging repository with the specific staging repository ID
exec_process ./gradlew releaseApacheStagingRepository --staging-repository-id "${STAGING_REPOSITORY_ID}"
+ cat <<EOT >> $GITHUB_STEP_SUMMARY
+ ## Nexus
+ ✅ Nexus staging repository released
+ EOT
+
+ - name: Release summary
+ run: |
cat <<EOT >> $GITHUB_STEP_SUMMARY
## Summary
🎉 Release published successfully:
@@ -462,3 +501,58 @@ jobs:
| Version | \`${version_without_rc}\` |
| Final release tag | \`${final_release_tag}\` |
EOT
+
+ publish-python-client:
+ name: Publish Python Client to PyPI
+ runs-on: ubuntu-latest
+ needs: [publish-release]
+ permissions:
+ contents: read
+ # IMPORTANT: this permission is mandatory for Trusted Publishing to PyPI
+ id-token: write
+ env:
+ version_without_rc: ${{ needs.publish-release.outputs.version_without_rc }}
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Set up environment variables
+ run: |
+ echo "RELEASEY_DIR=$(pwd)/releasey" >> $GITHUB_ENV
+ echo "LIBS_DIR=$(pwd)/releasey/libs" >> $GITHUB_ENV
+
+ - name: Install Subversion
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y subversion
+
+ - name: Download Python wheel from Apache dist release repository
+ env:
+ SVN_USERNAME: ${{ secrets.POLARIS_SVN_DEV_USERNAME }} # zizmor: ignore[secrets-outside-env]
+ SVN_PASSWORD: ${{ secrets.POLARIS_SVN_DEV_PASSWORD }} # zizmor: ignore[secrets-outside-env]
+ run: |
+ echo "::add-mask::$SVN_PASSWORD"
+
+ source "${LIBS_DIR}/_constants.sh"
+ source "${LIBS_DIR}/_exec.sh"
+
+ release_python_client_url="${APACHE_DIST_URL}/release/polaris/python-client/${version_without_rc}"
+
+ exec_process svn export --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \
+ "${release_python_client_url}" dist/
+
+ - name: Publish Python client release to PyPI
+ uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
+ with:
+ packages-dir: dist/
+
+ - name: Update step summary
+ run: |
+ cat <<EOT >> $GITHUB_STEP_SUMMARY
+ ## Python Client
+ ✅ Python client \`${version_without_rc}\` published to PyPI
+ EOT
diff --git a/Makefile b/Makefile
index b9b4d71620..e3e2bf27de 100644
--- a/Makefile
+++ b/Makefile
@@ -127,18 +127,21 @@ client-install-dependencies: $(VENV_DIR)
client-setup-env: $(VENV_DIR) client-install-dependencies
.PHONY: client-build
-client-build: client-setup-env ## Build client distribution. Pass FORMAT=sdist or FORMAT=wheel to build a specific format.
+client-build: client-setup-env ## Build client distribution. Pass FORMAT=sdist or FORMAT=wheel to build a specific format, and VERSION to stamp the version before building.
@echo "--- Building client distribution ---"
+ @if [ -n "$(VERSION)" ]; then \
+ $(ACTIVATE_AND_CD) && uv version "$(VERSION)"; \
+ fi
@if [ -n "$(FORMAT)" ]; then \
if [ "$(FORMAT)" != "sdist" ] && [ "$(FORMAT)" != "wheel" ]; then \
echo "Error: Invalid format '$(FORMAT)'. Supported formats are 'sdist' and 'wheel'." >&2; \
exit 1; \
fi; \
echo "Building with format: $(FORMAT)"; \
- $(ACTIVATE_AND_CD) && uv build --format $(FORMAT); \
+ $(ACTIVATE_AND_CD) && uv build --$(FORMAT); \
else \
echo "Building default distribution (sdist and wheel)"; \
- $(ACTIVATE_AND_CD) && uv build; \
+ $(ACTIVATE_AND_CD) && uv build --clear; \
fi
@echo "--- Client distribution build complete ---"
@@ -188,18 +191,27 @@ client-lint: client-setup-env ## Run linting checks for Polaris client
@$(ACTIVATE_AND_CD) && uv run --active pre-commit run --files integration_tests/* generate_clients.py apache_polaris/cli/* apache_polaris/cli/command/* apache_polaris/cli/options/* test/*
@echo "--- Client linting checks complete ---"
-.PHONY: client-nightly-publish
-client-nightly-publish: client-setup-env ## Build and publish nightly version to Test PyPI
- @echo "--- Starting nightly publish ---"
- @$(ACTIVATE_AND_CD) && \
- CURRENT_VERSION=$$(uv version --short) && \
- DATE_SUFFIX=$$(date -u +%Y%m%d%H%M%S) && \
- NIGHTLY_VERSION="$${CURRENT_VERSION}.dev$${DATE_SUFFIX}" && \
- echo "Publishing nightly version: $${NIGHTLY_VERSION}" && \
- uv version "$${NIGHTLY_VERSION}" && \
- uv build --clear && \
- uv publish --index testpypi
- @echo "--- Nightly publish complete ---"
+.PHONY: client-nightly-build
+client-nightly-build: client-setup-env ## Build nightly version for publishing to Test PyPI
+ @$(MAKE) client-build \
+ VERSION="$$($(VENV_DIR)/bin/uv --directory $(PYTHON_CLIENT_DIR) version --short).dev$$(date -u +%Y%m%d%H%M%S)" \
+ FORMAT=wheel
+
+.PHONY: client-rc-build
+client-rc-build: client-setup-env ## Build RC wheel for staging to Apache dist dev (requires RC_VERSION and RC_NUMBER)
+ @if [ -z "$(RC_VERSION)" ]; then echo "ERROR: RC_VERSION is not set"; exit 1; fi
+ @if [ -z "$(RC_NUMBER)" ]; then echo "ERROR: RC_NUMBER is not set"; exit 1; fi
+ @echo "--- Building RC wheel for version $(RC_VERSION) RC$(RC_NUMBER) ---"
+ @$(ACTIVATE_AND_CD) && uv version "$(RC_VERSION)" && uv build --wheel
+ @echo "--- RC wheel build complete ---"
+
+.PHONY: client-rc-testpypi-build
+client-rc-testpypi-build: client-setup-env ## Build RC wheel with rc suffix for Test PyPI (requires RC_VERSION and RC_NUMBER)
+ @if [ -z "$(RC_VERSION)" ]; then echo "ERROR: RC_VERSION is not set"; exit 1; fi
+ @if [ -z "$(RC_NUMBER)" ]; then echo "ERROR: RC_NUMBER is not set"; exit 1; fi
+ @echo "--- Building RC wheel for Test PyPI: $(RC_VERSION)rc$(RC_NUMBER) ---"
+ @$(ACTIVATE_AND_CD) && uv version "$(RC_VERSION)rc$(RC_NUMBER)" && uv build --wheel
+ @echo "--- RC wheel build for Test PyPI complete ---"
.PHONY: client-regenerate
client-regenerate: client-setup-env ## Regenerate the client code
diff --git a/client/python/pyproject.toml b/client/python/pyproject.toml
index d89cdf96f1..02cd4e4f60 100644
--- a/client/python/pyproject.toml
+++ b/client/python/pyproject.toml
@@ -19,12 +19,12 @@
[project]
name = "apache-polaris"
-version = "1.4.0"
+version = "1.4.0rc2"
description = "Apache Polaris"
authors = [{ name = "Apache Software Foundation", email = "dev@polaris.apache.org" }]
requires-python = ">=3.10,<3.14"
readme = "README.md"
-license = "Apache-2.0"
+license = {text = "Apache-2.0"}
license-files = ["LICENSE", "NOTICE"]
keywords = [
"Apache Polaris",
diff --git a/releasey/libs/_exec.sh b/releasey/libs/_exec.sh
index 3fd0e48b6b..9a12587416 100644
--- a/releasey/libs/_exec.sh
+++ b/releasey/libs/_exec.sh
@@ -70,13 +70,15 @@ function exec_process_with_retries {
function calculate_sha512 {
local source_file="$1"
+ local source_dir source_base
+ source_dir="$(dirname "${source_file}")"
+ source_base="$(basename "${source_file}")"
local target_file="${source_file}.sha512"
- # This function is only there for dry-run support. Because of the
- # redirection, we cannot use exec_process with the exact command that will be
- # executed.
+ # Run shasum from the file's directory so the checksum file contains only
+ # the filename, not the full path (required for standard sha512sum -c verification).
if [[ ${DRY_RUN:-1} -ne 1 ]]; then
- exec_process shasum -a 512 "${source_file}" > "${target_file}"
+ (cd "${source_dir}" && shasum -a 512 "${source_base}") > "${target_file}"
else
- exec_process "shasum -a 512 ${source_file} > ${target_file}"
+ exec_process "cd ${source_dir} && shasum -a 512 ${source_base} > ${target_file}"
fi
}