You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by li...@apache.org on 2022/12/21 00:06:04 UTC
[arrow-adbc] branch main updated: ci: run packaging builds on PRs (#255)
This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 8282b04 ci: run packaging builds on PRs (#255)
8282b04 is described below
commit 8282b04e86c8a83e2afbc6c7810354871449dfab
Author: David Li <li...@gmail.com>
AuthorDate: Tue Dec 20 19:05:59 2022 -0500
ci: run packaging builds on PRs (#255)
Fixes #252.
---
.github/workflows/packaging.yml | 121 +++++++++++++++++++------------------
ci/linux-packages/Rakefile | 2 +-
ci/scripts/java_jar_upload.sh | 43 +++++++++----
ci/scripts/python_wheel_upload.ps1 | 42 -------------
4 files changed, 95 insertions(+), 113 deletions(-)
diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml
index 5526580..c816b0d 100644
--- a/.github/workflows/packaging.yml
+++ b/.github/workflows/packaging.yml
@@ -18,6 +18,18 @@
name: Packaging
on:
+ pull_request:
+ branches:
+ - main
+ paths:
+ - "adbc.h"
+ - "c/**"
+ - "ci/**"
+ - "glib/**"
+ - "python/**"
+ - "ruby/**"
+ - ".github/workflows/packaging.yml"
+
push:
# Automatically build on RC tags
branches-ignore:
@@ -113,15 +125,6 @@ jobs:
dist/*.jar
dist/*.pom
- - name: Upload JARs to Gemfury
- shell: bash
- if: github.ref == 'refs/heads/main' && (github.event.schedule || inputs.upload_artifacts)
- run: |
- # Preserve env vars so GEMFURY_PUSH_TOKEN gets preserved
- sudo -E ./ci/scripts/java_jar_upload.sh "$(pwd)"
- env:
- GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
-
linux:
name: Linux ${{ matrix.target }}
runs-on: ubuntu-latest
@@ -232,7 +235,12 @@ jobs:
matrix:
arch: ["amd64", "arm64v8"]
manylinux_version: ["2014"]
- # Let this one run in parallel because the arm64 build is so much slower
+ is_pr:
+ - ${{ startsWith(github.ref, 'refs/pull/') }}
+ exclude:
+ # Don't run arm64v8 build on PRs since the build is excessively slow
+ - arch: arm64v8
+ is_pr: true
steps:
- uses: actions/checkout@v3
with:
@@ -282,22 +290,12 @@ jobs:
env PYTHON=3.10 docker-compose run python-wheel-manylinux-test
env PYTHON=3.11 docker-compose run python-wheel-manylinux-test
- - name: Upload wheels to Gemfury
- shell: bash
- if: github.ref == 'refs/heads/main' && (github.event.schedule || github.event.inputs.upload_artifacts == true)
- run: |
- ./ci/scripts/python_wheel_upload.sh python/adbc_driver_{manager,postgresql,sqlite}/repaired_wheels/*.whl
- env:
- GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
-
python-macos:
name: "Python ${{ matrix.arch }} macOS"
runs-on: macos-latest
strategy:
matrix:
arch: ["amd64", "arm64v8"]
- # Limit parallelism; gemfury appears to return 409 CONFLICT on concurrent uploads
- max-parallel: 1
env:
MACOSX_DEPLOYMENT_TARGET: "10.15"
PYTHON: "/Library/Frameworks/Python.framework/Versions/3.10/bin/python3.10"
@@ -383,28 +381,12 @@ jobs:
./ci/scripts/python_wheel_unix_test.sh $(pwd)
deactivate
- - name: Upload wheels to Gemfury
- shell: bash
- if: github.ref == 'refs/heads/main' && (github.event.schedule || github.event.inputs.upload_artifacts == true)
- run: |
- ./ci/scripts/python_wheel_upload.sh python/adbc_driver_{manager,postgresql,sqlite}/repaired_wheels/*.whl
- env:
- GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
-
python-windows:
name: "Python ${{ matrix.python_version }} Windows"
runs-on: windows-latest
strategy:
matrix:
- python_version: ["3.9.13", "3.10.8", "3.11.0"]
- include:
- - python_version: "3.9.13"
- PYTHON_HOME: "C:\\Python39"
- - python_version: "3.10.8"
- PYTHON_HOME: "C:\\Python310"
-
- # Limit parallelism; gemfury appears to return 409 CONFLICT on concurrent uploads
- max-parallel: 1
+ python_version: ["3.9", "3.10", "3.11"]
env:
PYTHON_VERSION: "${{ matrix.python_version }}"
# Where to install vcpkg
@@ -448,10 +430,9 @@ jobs:
popd
- name: Install Python ${{ matrix.python_version }}
- shell: pwsh
- run: |
- choco install --no-progress -y python --version=${{ matrix.python_version }}
- echo ("PATH=${{ matrix.PYTHON_HOME }};" + $env:PATH) >> $env:GITHUB_ENV
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ matrix.python_version }}
- name: Build wheel
shell: cmd
@@ -478,16 +459,6 @@ jobs:
CALL venv\Scripts\activate.bat
.\ci\scripts\python_wheel_windows_test.bat %cd%
- - name: Upload wheels to Gemfury
- shell: pwsh
- if: github.ref == 'refs/heads/main' && (github.event.schedule || inputs.upload_artifacts)
- run: |
- foreach ($wheel in Get-ChildItem python\adbc_driver_*\repaired_wheels\*.whl) {
- .\ci\scripts\python_wheel_upload.ps1 $wheel.FullName
- }
- env:
- GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
-
python-sdist:
name: "Python sdist"
runs-on: ubuntu-latest
@@ -529,14 +500,6 @@ jobs:
run: |
docker-compose run python-sdist-test
- - name: Upload sdist to Gemfury
- shell: bash
- if: github.ref == 'refs/heads/main' && (github.event.schedule || github.event.inputs.upload_artifacts == true || github.event.inputs.upload_artifacts == 'true')
- run: |
- ./ci/scripts/python_wheel_upload.sh python/adbc_driver_{manager,postgresql,sqlite}/dist/*.tar.gz
- env:
- GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
-
release:
name: "Create release"
runs-on: ubuntu-latest
@@ -583,3 +546,43 @@ jobs:
${UPLOAD}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ upload-gemfury:
+ name: "Upload packages to Gemfury"
+ runs-on: ubuntu-latest
+ if: github.ref == 'refs/heads/main' && (github.event.schedule || inputs.upload_artifacts)
+ needs:
+ - java
+ - python-manylinux
+ - python-macos
+ - python-windows
+ - python-sdist
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+ persist-credentials: true
+ - name: Get All Artifacts
+ uses: actions/download-artifact@v3
+ with:
+ path: nightly-artifacts
+ - name: Upload
+ shell: bash
+ run: |
+ # Deduplicate wheels built in different jobs with same tag
+ mkdir -p upload-staging
+ find ./nightly-artifacts/ \
+ '(' \
+ -name '*.jar' -or \
+ -name '*.pom' -or \
+ -name '*.whl' -or \
+ -name 'adbc_*.tar.gz' \
+ ')' \
+ -exec mv '{}' upload-staging \;
+
+ # Java
+ ./ci/scripts/java_jar_upload.sh upload-staging/*.pom
+ # Python
+ ./ci/scripts/python_wheel_upload.sh upload-staging/adbc_*.tar.gz upload-staging/*.whl
+ env:
+ GEMFURY_PUSH_TOKEN: foo
diff --git a/ci/linux-packages/Rakefile b/ci/linux-packages/Rakefile
index 7fcc272..bb281ed 100644
--- a/ci/linux-packages/Rakefile
+++ b/ci/linux-packages/Rakefile
@@ -38,7 +38,7 @@ module Helper
version_env = ENV["VERSION"]
return version_env if version_env
- meson_build = top_source_dir / "c" / "cmake_modules" / "AdbcDefines.cmake"
+ meson_build = top_source_dir / "c" / "cmake_modules" / "AdbcVersion.cmake"
version = meson_build.read.scan(/ADBC_VERSION "(.+?)"/)[0][0]
formatted_release_time = release_time.strftime("%Y%m%d")
version.gsub(/-SNAPSHOT\z/) {"-dev#{formatted_release_time}"}
diff --git a/ci/scripts/java_jar_upload.sh b/ci/scripts/java_jar_upload.sh
index 74b809f..702b3b3 100755
--- a/ci/scripts/java_jar_upload.sh
+++ b/ci/scripts/java_jar_upload.sh
@@ -18,9 +18,10 @@
set -ex
+# Usage: java_jar_upload.sh jar1.pom jar2.pom ...
+
main() {
- local -r source_dir="${1}"
- local settings_file=$(mktemp adbc.settingsXXXXXXXX.xml --tmpdir)
+ local settings_file=$(mktemp adbc.settingsXXXXXXXX)
trap 'rm -f "$settings_file"' ERR EXIT INT TERM
if [[ -z "${GEMFURY_PUSH_TOKEN}" ]]; then
@@ -28,8 +29,6 @@ main() {
exit 1
fi
- pushd "${source_dir}/java"
-
cat <<SETTINGS > "${settings_file}"
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
@@ -51,14 +50,36 @@ main() {
</settings>
SETTINGS
- mvn \
- -Dmaven.install.skip=true \
- -Drat.skip=true \
- -DskipTests \
- --settings "${settings_file}" \
- deploy
+ for pom in "$@"; do
+ echo "Deploying ${pom}"
+ local mvnArgs=""
+
+ local filename=$(basename "${pom}" .pom)
+ local jar=$(dirname "${pom}")/"${filename}.jar"
+ local sources=$(dirname "${pom}")/"${filename}-javadoc.jar"
+ local javadoc=$(dirname "${pom}")/"${filename}-sources.jar"
+
+ if [[ -f "${sources}" ]]; then
+ mvnArgs="${mvnArgs} -Dsources=${sources}"
+ fi
+
+ if [[ -f "${javadoc}" ]]; then
+ mvnArgs="${mvnArgs} -Djavadoc=${javadoc}"
+ fi
- popd
+ mvn \
+ -Dmaven.install.skip=true \
+ -Drat.skip=true \
+ -DskipTests \
+ --settings "${settings_file}" \
+ deploy:deploy-file \
+ -DrepositoryId=fury \
+ -Durl=https://maven.fury.io/arrow-adbc-nightlies/ \
+ -DgeneratePom=false \
+ -Dfile="${jar}" \
+ -DpomFile="${pom}" \
+ ${mvnArgs}
+ done
}
main "$@"
diff --git a/ci/scripts/python_wheel_upload.ps1 b/ci/scripts/python_wheel_upload.ps1
deleted file mode 100644
index 82284e9..0000000
--- a/ci/scripts/python_wheel_upload.ps1
+++ /dev/null
@@ -1,42 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-$Uri = "https://push.fury.io/arrow-adbc-nightlies/"
-
-for ($i = 0; $i -lt $args.count; $i++) {
- echo "Uploading $($args[$i])"
- $Token = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("$($env:GEMFURY_PUSH_TOKEN):"))
- $Form = @{
- package = Get-Item -Path $args[$i]
- }
- $Headers = @{
- Authorization = "Basic $($Token)"
- }
- try {
- $Response = Invoke-WebRequest -uri $Uri -Method Post -Form $Form -Headers $Headers
- echo "Uploaded $($args[$i])"
- } catch {
- $StatusCode = $_.Exception.Response.StatusCode.value__
- if ($StatusCode -eq 409) {
- echo "Already uploaded $($args[$i])"
- continue
- } else {
- echo "Failed to upload: $($StatusCode) $($_.Exception)"
- exit 1
- }
- }
-}