You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ma...@apache.org on 2024/07/02 15:32:52 UTC
(flink) 01/02: [FLINK-34487][ci] Adds Python Wheels nightly GHA workflow
This is an automated email from the ASF dual-hosted git repository.
mapohl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit ab856fc6c8b1cdd402c11f7755b01a51aafd8f4c
Author: morazow <m....@gmail.com>
AuthorDate: Fri Mar 1 11:38:36 2024 +0100
[FLINK-34487][ci] Adds Python Wheels nightly GHA workflow
Co-authored-by: Matthias Pohl <gi...@mapohl.com>
---
.github/actions/stringify/action.yml | 42 +++++++++++++++++++++++++++++
.github/workflows/nightly.yml | 48 +++++++++++++++++++++++++++++++++
.github/workflows/template.flink-ci.yml | 15 ++++-------
3 files changed, 95 insertions(+), 10 deletions(-)
diff --git a/.github/actions/stringify/action.yml b/.github/actions/stringify/action.yml
new file mode 100644
index 00000000000..e05547f78dd
--- /dev/null
+++ b/.github/actions/stringify/action.yml
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: "Stringify"
+description: "Stringifies the given input value."
+inputs:
+ value:
+ description: "Input value to stringify."
+ required: true
+outputs:
+ stringified_value:
+ description: "Stringified output value."
+ value: ${{ steps.stringify-step.outputs.stringified_value }}
+runs:
+ using: "composite"
+ steps:
+ - name: "Stringify '${{ inputs.value }}'"
+ id: stringify-step
+ shell: bash
+ run: |
+ # adds a stringified version of the workflow name that can be used for generating unique build artifact names within a composite workflow
+ # - replaces any special characters (except for underscores and dots) with dashes
+ # - makes the entire string lowercase
+ # - condenses multiple dashes into a single one
+ # - removes leading and following dashes
+ stringified_value=$(echo "${{ inputs.value }}" | tr -C '[:alnum:]._' '-' | tr '[:upper:]' '[:lower:]' | sed -e 's/--*/-/g' -e 's/^-*//g' -e 's/-*$//g')
+ echo "stringified_value=${stringified_value}" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 2090a3b1156..67994be49a5 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -94,3 +94,51 @@ jobs:
s3_bucket: ${{ secrets.IT_CASE_S3_BUCKET }}
s3_access_key: ${{ secrets.IT_CASE_S3_ACCESS_KEY }}
s3_secret_key: ${{ secrets.IT_CASE_S3_SECRET_KEY }}
+
+ build_python_wheels:
+ name: "Build Python Wheels on ${{ matrix.os }}"
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - os: ubuntu-latest
+ os_name: linux
+ python-version: 3.9
+ - os: macos-latest
+ os_name: macos
+ python-version: 3.9
+ steps:
+ - name: "Checkout the repository"
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+ - name: "Install python"
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: "Stringify workflow name"
+ uses: "./.github/actions/stringify"
+ id: stringify_workflow
+ with:
+ value: ${{ github.workflow }}
+ - name: "Build python wheels for ${{ matrix.os_name }}"
+ if: matrix.os_name == 'linux'
+ run: |
+ cd flink-python
+ bash dev/build-wheels.sh
+ - name: "Build python wheels for ${{ matrix.os }}"
+ if: matrix.os_name == 'macos'
+ run: |
+ cd flink-python
+ python -m pip install --upgrade pip
+ pip install cibuildwheel==2.8.0
+ cibuildwheel --platform macos --output-dir dist .
+ - name: "Tar python artifact"
+ run: tar -czvf python-wheel-${{ matrix.os_name }}.tar.gz -C flink-python/dist .
+ - name: "Upload python artifact"
+ uses: actions/upload-artifact@v4
+ with:
+ name: wheel_${{ matrix.os_name }}_${{ steps.stringify_workflow.outputs.stringified_value }}-${{ github.run_number }}
+ path: python-wheel-${{ matrix.os_name }}.tar.gz
diff --git a/.github/workflows/template.flink-ci.yml b/.github/workflows/template.flink-ci.yml
index 1ac63916ddd..2b97c11a533 100644
--- a/.github/workflows/template.flink-ci.yml
+++ b/.github/workflows/template.flink-ci.yml
@@ -80,7 +80,7 @@ jobs:
# timeout in minutes - this environment variable is required by uploading_watchdog.sh
GHA_JOB_TIMEOUT: 240
outputs:
- stringified-workflow-name: ${{ steps.workflow-prep-step.outputs.stringified-workflow-name }}
+ stringified-workflow-name: ${{ steps.workflow-prep-step.outputs.stringified_value }}
steps:
- name: "Flink Checkout"
uses: actions/checkout@v4
@@ -96,15 +96,10 @@ jobs:
target_directory: ${{ env.CONTAINER_LOCAL_WORKING_DIR }}
- name: "Stringify workflow name"
+ uses: "./.github/actions/stringify"
id: workflow-prep-step
- run: |
- # adds a stringified version of the workflow name that can be used for generating unique build artifact names within a composite workflow
- # - replaces any special characters (except for underscores and dots) with dashes
- # - makes the entire string lowercase
- # - condenses multiple dashes into a single one
- # - removes leading and following dashes
- stringified_workflow_name=$(echo "${{ github.workflow }}-${{ inputs.workflow-caller-id }}" | tr -C '[:alnum:]._' '-' | tr '[:upper:]' '[:lower:]' | sed -e 's/--*/-/g' -e 's/^-*//g' -e 's/-*$//g')
- echo "stringified-workflow-name=${stringified_workflow_name}" >> $GITHUB_OUTPUT
+ with:
+ value: "${{ github.workflow }}-${{ inputs.workflow-caller-id }}"
- name: "Compile Flink"
uses: "./.github/actions/run_mvn"
@@ -122,7 +117,7 @@ jobs:
- name: "Upload artifacts to make them available in downstream jobs"
uses: actions/upload-artifact@v4
with:
- name: build-artifacts-${{ steps.workflow-prep-step.outputs.stringified-workflow-name }}-${{ github.run_number }}
+ name: build-artifacts-${{ steps.workflow-prep-step.outputs.stringified_value }}-${{ github.run_number }}
path: ${{ env.FLINK_ARTIFACT_DIR }}/${{ env.FLINK_ARTIFACT_FILENAME }}
if-no-files-found: error
# use minimum here because we only need these artifacts to speed up the build