You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by xt...@apache.org on 2025/10/08 03:39:10 UTC
(flink-agents) 02/02: [ci] Add release scripts
This is an automated email from the ASF dual-hosted git repository.
xtsong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-agents.git
commit 53d29e6fcf418c7625a04466cc6c4cc847ac3262
Author: sxnan <su...@gmail.com>
AuthorDate: Tue Oct 7 20:39:01 2025 +0800
[ci] Add release scripts
---
.github/workflows/build_wheel.yml | 156 +++++++++++++++++++++++++++++++
.gitignore | 1 +
pom.xml | 145 ++++++++++++++++++++++++++++
tools/build.sh | 3 +-
tools/releasing/create_binary_release.sh | 111 ++++++++++++++++++++++
tools/releasing/create_release_branch.sh | 76 +++++++++++++++
tools/releasing/create_source_release.sh | 82 ++++++++++++++++
tools/releasing/deploy_staging_jars.sh | 46 +++++++++
8 files changed, 619 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml
new file mode 100644
index 0000000..6b9d153
--- /dev/null
+++ b/.github/workflows/build_wheel.yml
@@ -0,0 +1,156 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Build Wheel
+
+on:
+ workflow_dispatch:
+
+jobs:
+ lint:
+ name: Code Style Check
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - name: Check License Header
+ run: tools/check-license.sh
+ - name: Set up JDK
+ uses: actions/setup-java@v4
+ with:
+ java-version: '11'
+ distribution: 'adopt'
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: 3.11
+ - name: Install uv
+ uses: astral-sh/setup-uv@v4
+ with:
+ version: "latest"
+ - name: Check code style
+ run: ./tools/lint.sh -c
+
+ java_tests:
+ name: java tests on ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ 'macos-latest', 'ubuntu-latest' ]
+ steps:
+ - uses: actions/checkout@v4
+ - name: Install java
+ uses: actions/setup-java@v4
+ with:
+ java-version: '11'
+ distribution: 'adopt'
+ - name: Run Java Tests
+ run: tools/ut.sh -j
+
+ python_tests:
+ name: python tests on ${{ matrix.os }} ${{ matrix.python-version}}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ 'macos-latest', 'ubuntu-latest' ]
+ python-version: [ '3.10', '3.11' ]
+ steps:
+ - uses: actions/checkout@v4
+ - name: Install java
+ uses: actions/setup-java@v4
+ with:
+ java-version: '11'
+ distribution: 'adopt'
+ - name: Install python
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install uv
+ uses: astral-sh/setup-uv@v4
+ with:
+ version: "latest"
+ - name: Run Python Tests
+ run: tools/ut.sh -p
+
+ e2e_tests:
+ name: e2e tests on ${{ matrix.os }} ${{ matrix.python-version}}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ 'macos-latest', 'ubuntu-latest' ]
+ python-version: [ '3.10', '3.11' ]
+ steps:
+ - uses: actions/checkout@v4
+ - name: Install java
+ uses: actions/setup-java@v4
+ with:
+ java-version: '11'
+ distribution: 'adopt'
+ - name: Install python
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install uv
+ uses: astral-sh/setup-uv@v4
+ with:
+ version: "latest"
+ - name: Build flink-agents
+ run: bash tools/build.sh
+ - name: Run e2e tests
+ run: bash tools/e2e.sh
+
+ build_python_wheels:
+ name: "Build Python Wheels"
+ runs-on: ubuntu-latest
+ steps:
+ - name: "Checkout the repository"
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+ - name: Install java
+ uses: actions/setup-java@v4
+ with:
+ java-version: '11'
+ distribution: 'adopt'
+ - name: Install python
+ uses: actions/setup-python@v4
+ with:
+ python-version: 3.11
+ - name: Install uv
+ uses: astral-sh/setup-uv@v4
+ with:
+ version: "latest"
+ - name: "Build python wheels"
+ run: bash tools/build.sh
+ env:
+ # Skip -musllinux on Linux builds
+ CIBW_SKIP: "*-musllinux*"
+ # Use manylinux2014 on Linux
+ CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+ CIBW_BEFORE_ALL_LINUX: pip install patchelf
+ CIBW_BEFORE_BUILD_LINUX: pip install --upgrade pip && pip install --group python/pyproject.toml:dev
+ CIBW_ENVIRONMENT_LINUX: CFLAGS="-I. -include ./dev/glibc_version_fix.h"
+ # Run auditwheel repair on Linux
+ CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair -w {dest_dir} {wheel}"
+ - name: "Upload python wheels"
+ uses: actions/upload-artifact@v4
+ with:
+ name: wheel-${{ github.run_number }}
+ path: python/dist/**
diff --git a/.gitignore b/.gitignore
index c34bd50..345b3c2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@ python/uv.lock
python/.DS_Store
__pycache__
**/dependency-reduced-pom.xml
+/tools/releasing/release/
diff --git a/pom.xml b/pom.xml
index b4bf41a..d78e2a6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -51,6 +51,9 @@ under the License.
<slf4j.version>1.7.36</slf4j.version>
<assertj.version>3.23.1</assertj.version>
<mockito.version>5.8.0</mockito.version>
+
+ <gpg.useagent>true</gpg.useagent>
+ <arguments />
</properties>
<modules>
@@ -84,7 +87,149 @@ under the License.
</dependency>
</dependencies>
+ <profiles>
+ <profile>
+ <id>release</id>
+ <activation>
+ <property>
+ <name>release</name>
+ </property>
+ </activation>
+ <properties>
+ <target.java.version>11</target.java.version>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-gpg-plugin</artifactId>
+ <version>1.4</version>
+ <executions>
+ <execution>
+ <id>sign-artifacts</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>sign</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>enforce-maven</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <!-- versions for certain build tools are enforced to match the CI setup -->
+ <!-- the rules below should stay in sync with Flink Release wiki documentation and the CI scripts -->
+ <requireJavaVersion>
+ <version>[${target.java.version}.*,)</version>
+ </requireJavaVersion>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>attach-javadocs</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-release-plugin</artifactId>
+ <version>2.1</version>
+ <configuration>
+ <mavenExecutorId>forked-path</mavenExecutorId>
+ <useReleaseProfile>false</useReleaseProfile>
+ <arguments>${arguments} -Psonatype-oss-release</arguments>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+ </profile>
+
+ <profile>
+ <!-- used for SNAPSHOT and regular releases -->
+ <id>docs-and-source</id>
+ <activation>
+ <property>
+ <name>docs-and-source</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>3.2.1</version>
+ <executions>
+ <execution>
+ <id>attach-sources</id>
+ <goals>
+ <goal>jar-no-fork</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <quiet>true</quiet>
+ </configuration>
+ <executions>
+ <execution>
+ <id>attach-javadocs</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
<build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>3.8.0</version>
+ <configuration>
+ <quiet>true</quiet>
+ <detectOfflineLinks>false</detectOfflineLinks>
+ <release>11</release>
+ <additionalJOptions>
+ <additionalJOption>-Xdoclint:none</additionalJOption>
+ <additionalJOption>--allow-script-in-comments</additionalJOption>
+ <!-- Suppress the error that is accepted by JDK 8 but not by JDK 11. -->
+ <additionalJOption>--ignore-source-errors</additionalJOption>
+ </additionalJOptions>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>com.diffplug.spotless</groupId>
diff --git a/tools/build.sh b/tools/build.sh
index ab35cad..baab628 100755
--- a/tools/build.sh
+++ b/tools/build.sh
@@ -37,7 +37,8 @@ PROJECT_ROOT="${BASE_DIR}/../"
# build java
if $build_java; then
- mvn clean package -DskipTests
+ mvn --version
+ mvn clean package -DskipTests -B
fi
# copy flink-agents-dist jar to python lib
diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh
new file mode 100755
index 0000000..41e873d
--- /dev/null
+++ b/tools/releasing/create_binary_release.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##
+## Variables with defaults (if not overwritten by environment)
+##
+SKIP_GPG=${SKIP_GPG:-false}
+MVN=${MVN:-mvn}
+
+if [ -z "${RELEASE_VERSION:-}" ]; then
+ echo "RELEASE_VERSION was not set."
+ exit 1
+fi
+
+# fail immediately
+set -o errexit
+set -o nounset
+# print command before executing
+set -o xtrace
+
+CURR_DIR=`pwd`
+if [[ `basename $CURR_DIR` != "tools" ]] ; then
+ echo "You have to call the script from the tools/ dir"
+ exit 1
+fi
+
+if [ "$(uname)" == "Darwin" ]; then
+ SHASUM="shasum -a 512"
+else
+ SHASUM="sha512sum"
+fi
+
+cd ..
+
+FLINK_AGENTS_DIR=`pwd`
+RELEASE_DIR=${FLINK_AGENTS_DIR}/tools/releasing/release
+PYTHON_RELEASE_DIR=${RELEASE_DIR}/python
+mkdir -p ${PYTHON_RELEASE_DIR}
+
+###########################
+
+# build maven package, create Flink distribution, generate signature
+make_binary_release() {
+
+ echo "Creating binary release"
+
+ DIST_JAR_NAME="flink-agents-dist-${RELEASE_VERSION}.jar"
+
+ # enable release profile here (to check for the maven version)
+ $MVN clean package -Prelease -pl dist -am -Dgpg.skip -Dcheckstyle.skip=true -DskipTests
+
+ cd dist/target
+ cp $DIST_JAR_NAME ${RELEASE_DIR}
+ cd ${RELEASE_DIR}
+
+ # Sign sha the tgz
+ if [ "$SKIP_GPG" == "false" ] ; then
+ gpg --armor --detach-sig "${DIST_JAR_NAME}"
+ fi
+ $SHASUM "${DIST_JAR_NAME}" > "${DIST_JAR_NAME}.sha512"
+
+ cd ${FLINK_AGENTS_DIR}
+}
+
+make_python_release() {
+ FLINK_AGENTS_VERSION=${RELEASE_VERSION/-SNAPSHOT/.dev0}
+ cd python/dist
+
+ # Need to move the downloaded wheel packages from Azure CI to the directory flink-python/dist manually.
+ for wheel_file in *.whl; do
+ if [[ ! ${wheel_file} =~ ^flink_agents-$FLINK_AGENTS_VERSION- ]]; then
+ echo -e "\033[31;1mThe file name of the python package: ${wheel_file} is not consistent with given release version: ${FLINK_AGENTS_VERSION}!\033[0m"
+ exit 1
+ fi
+ cp ${wheel_file} "${PYTHON_RELEASE_DIR}/${wheel_file}"
+ done
+
+ cd ${PYTHON_RELEASE_DIR}
+
+ # Sign sha the tgz and wheel packages
+ if [ "$SKIP_GPG" == "false" ] ; then
+ for wheel_file in *.whl; do
+ gpg --armor --detach-sig "${wheel_file}"
+ done
+ fi
+
+ for wheel_file in *.whl; do
+ $SHASUM "${wheel_file}" > "${wheel_file}.sha512"
+ done
+
+ cd ${FLINK_AGENTS_DIR}
+}
+
+make_binary_release
+make_python_release
diff --git a/tools/releasing/create_release_branch.sh b/tools/releasing/create_release_branch.sh
new file mode 100755
index 0000000..ebe7683
--- /dev/null
+++ b/tools/releasing/create_release_branch.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##
+## Variables with defaults (if not overwritten by environment)
+##
+RELEASE_CANDIDATE=${RELEASE_CANDIDATE:-none}
+MVN=${MVN:-mvn}
+
+if [ -z "${NEW_VERSION:-}" ]; then
+ echo "NEW_VERSION was not set."
+ exit 1
+fi
+
+# fail immediately
+set -o errexit
+set -o nounset
+# print command before executing
+set -o xtrace
+
+CURR_DIR=`pwd`
+if [[ `basename $CURR_DIR` != "tools" ]] ; then
+ echo "You have to call the script from the tools/ dir"
+ exit 1
+fi
+
+###########################
+
+cd ..
+
+target_branch=release-$NEW_VERSION
+if [ "$RELEASE_CANDIDATE" != "none" ]; then
+ target_branch=$target_branch-rc$RELEASE_CANDIDATE
+fi
+
+git checkout -b $target_branch
+
+#change version in all pom files
+$MVN org.codehaus.mojo:versions-maven-plugin:2.8.1:set -DnewVersion=$NEW_VERSION -DgenerateBackupPoms=false
+
+#change version of documentation
+cd docs
+perl -pi -e "s#^ Version = .*# Version = \"${NEW_VERSION}\"#" config.toml
+
+# The version in the title should not contain the bugfix version (e.g. 1.3)
+VERSION_TITLE=$(echo $NEW_VERSION | sed 's/\.[^.]*$//')
+perl -pi -e "s#^ VersionTitle = .*# VersionTitle = \"${VERSION_TITLE}\"#" config.toml
+cd ..
+
+#change version of flink-agents
+cd python
+perl -pi -e "s#^version = \".*\"#version = \"${NEW_VERSION}\"#" pyproject.toml
+cd ..
+
+git commit -am "Commit for release $NEW_VERSION"
+
+RELEASE_HASH=`git rev-parse HEAD`
+echo "Echo created release hash $RELEASE_HASH"
+
+echo "Done. Don't forget to create the signed release tag at the end and push the changes."
diff --git a/tools/releasing/create_source_release.sh b/tools/releasing/create_source_release.sh
new file mode 100755
index 0000000..b64519f
--- /dev/null
+++ b/tools/releasing/create_source_release.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##
+## Variables with defaults (if not overwritten by environment)
+##
+MVN=${MVN:-mvn}
+
+if [ -z "${RELEASE_VERSION:-}" ]; then
+ echo "RELEASE_VERSION was not set."
+ exit 1
+fi
+
+# fail immediately
+set -o errexit
+set -o nounset
+# print command before executing
+set -o xtrace
+
+CURR_DIR=`pwd`
+if [[ `basename $CURR_DIR` != "tools" ]] ; then
+ echo "You have to call the script from the tools/ dir"
+ exit 1
+fi
+
+if [ "$(uname)" == "Darwin" ]; then
+ SHASUM="shasum -a 512"
+else
+ SHASUM="sha512sum"
+fi
+
+###########################
+
+cd ..
+
+FLINK_AGENTS=`pwd`
+RELEASE_DIR=${FLINK_AGENTS}/tools/releasing/release
+CLONE_DIR=${RELEASE_DIR}/flink-tmp-clone
+
+echo "Creating source package"
+
+mkdir -p ${RELEASE_DIR}
+
+# create a temporary git clone to ensure that we have a pristine source release
+git clone ${FLINK_AGENTS} ${CLONE_DIR}
+cd ${CLONE_DIR}
+
+rsync -a \
+ --exclude ".git" --exclude ".gitignore" --exclude ".gitattributes" --exclude "azure-pipelines.yml" --exclude ".asf.yaml" \
+ --exclude "CHANGELOG" --exclude ".github" --exclude "target" \
+ --exclude ".idea" --exclude "*.iml" --exclude ".DS_Store" --exclude "build-target" \
+ --exclude "docs/public" --exclude "docs/resources" --exclude "docs/themes" \
+ --exclude "*/dependency-reduced-pom.xml" \
+ --exclude "python/lib" --exclude "python/build" \
+ --exclude "python/dist" --exclude "python/flink_agents.egg-info" \
+ --exclude "python/.tox" --exclude "python/.cache" --exclude "python/uv.lock" \
+ --exclude "python/.pytest_cache" \
+ . flink-agents-$RELEASE_VERSION
+
+tar czf ${RELEASE_DIR}/flink-agents-${RELEASE_VERSION}-src.tgz flink-agents-$RELEASE_VERSION
+gpg --armor --detach-sig ${RELEASE_DIR}/flink-agents-$RELEASE_VERSION-src.tgz
+cd ${RELEASE_DIR}
+$SHASUM flink-agents-$RELEASE_VERSION-src.tgz > flink-agents-$RELEASE_VERSION-src.tgz.sha512
+
+cd ${CURR_DIR}
+rm -rf ${CLONE_DIR}
diff --git a/tools/releasing/deploy_staging_jars.sh b/tools/releasing/deploy_staging_jars.sh
new file mode 100755
index 0000000..e076869
--- /dev/null
+++ b/tools/releasing/deploy_staging_jars.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##
+## Variables with defaults (if not overwritten by environment)
+##
+MVN=${MVN:-mvn}
+CUSTOM_OPTIONS=${CUSTOM_OPTIONS:-}
+
+# fail immediately
+set -o errexit
+set -o nounset
+# print command before executing
+set -o xtrace
+
+CURR_DIR=`pwd`
+if [[ `basename $CURR_DIR` != "tools" ]] ; then
+ echo "You have to call the script from the tools/ dir"
+ exit 1
+fi
+
+###########################
+
+cd ..
+
+echo "Deploying to repository.apache.org"
+
+COMMON_OPTIONS="-Prelease,docs-and-source -DskipTests -DretryFailedDeploymentCount=10 $CUSTOM_OPTIONS"
+
+$MVN clean deploy $COMMON_OPTIONS