You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mb...@apache.org on 2022/10/27 19:08:17 UTC
[flink] 26/30: [apple] Add binary compatibility check
This is an automated email from the ASF dual-hosted git repository.
mbalassi pushed a commit to branch release-1.16.0.1-acs
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 375b136807e938498440cdd3c07a90c32a8a35bc
Author: Gabor Somogyi <ga...@apple.com>
AuthorDate: Thu Oct 13 19:07:43 2022 +0200
[apple] Add binary compatibility check
---
apple/README.md | 11 +++
apple/create_release_branch.sh | 19 ++++--
apple/release_rio_bump.patch.template | 72 --------------------
apple/update_rio_for_release.py | 78 ++++++++++++++++++++++
.../flink-sql-connector-hive-3.1.3.0-apple/pom.xml | 7 ++
rio.yml | 2 +-
6 files changed, 112 insertions(+), 77 deletions(-)
diff --git a/apple/README.md b/apple/README.md
index f4de683046c..834f63c0ec9 100644
--- a/apple/README.md
+++ b/apple/README.md
@@ -18,6 +18,17 @@ MAJOR_VERSION=1
MINOR_VERSION=15
PATCH_VERSION=1
ACS_VERSION=0
+# BCC = binary compatibility check
+# BCC version must be the previous patch version of a release
+BCC_MAJOR_VERSION=1
+BCC_MINOR_VERSION=15
+BCC_PATCH_VERSION=0
+BCC_ACS_VERSION=0
```
The script assumes you are currently on the branch `release-$MAJOR_VERSION.$MINOR_VERSION-acs` and the current snapshot version is `$MAJOR_VERSION.$MINOR_VERSION.$PATCH_VERSION-acs-SNAPSHOT`.
+
+Binary compatibility is enforced between patch releases from version `1.15.2.2-acs`.
+This effectively means binary compatibility will be checked between the following versions:
+* $MAJOR_VERSION.$MINOR_VERSION.$PATCH_VERSION.$ACS_VERSION-acs
+* $BCC_MAJOR_VERSION.$BCC_MINOR_VERSION.$BCC_PATCH_VERSION.$BCC_ACS_VERSION-acs
diff --git a/apple/create_release_branch.sh b/apple/create_release_branch.sh
index c7aee6dc08a..69f42423042 100755
--- a/apple/create_release_branch.sh
+++ b/apple/create_release_branch.sh
@@ -22,12 +22,18 @@ MAJOR_VERSION=1
MINOR_VERSION=15
PATCH_VERSION=1
ACS_VERSION=0
+# BCC = binary compatibility check
+# BCC version must be the previous patch version of a release
+BCC_MAJOR_VERSION=1
+BCC_MINOR_VERSION=15
+BCC_PATCH_VERSION=0
+BCC_ACS_VERSION=0
OLD="$MAJOR_VERSION.$MINOR_VERSION.$PATCH_VERSION-acs-SNAPSHOT"
RELEASE_VERSION="$MAJOR_VERSION.$MINOR_VERSION.$PATCH_VERSION.$ACS_VERSION-acs"
-BRANCH_NAME="release-$MAJOR_VERSION.$MINOR_VERSION-acs"
+BCC_RELEASE_VERSION="$BCC_MAJOR_VERSION.$BCC_MINOR_VERSION.$BCC_PATCH_VERSION.$BCC_ACS_VERSION-acs"
-HERE=` basename "$PWD"`
+HERE=$(basename "$PWD")
if [[ "$HERE" != "apple" ]]; then
echo "Please only execute in the apple/ directory";
exit 1;
@@ -41,10 +47,15 @@ find .. -name 'pom.xml' -type f -exec perl -pi -e 's#<version>'"$OLD"'</version>
# change version of the quickstart property
find .. -name 'pom.xml' -type f -exec perl -pi -e 's#<flink.version>'"$OLD"'</flink.version>#<flink.version>'"$RELEASE_VERSION"'</flink.version>#' {} \;
-git commit -am"[apple][release] Set release version to $RELEASE_VERSION"
+git commit -am"[apple][release] Set release version to $RELEASE_VERSION with binary compatibility to $BCC_RELEASE_VERSION"
cd ..
-sed -e "s/\${MINOR_VERSION}/${MINOR_VERSION}/" -e "s/\${PATCH_VERSION}/${PATCH_VERSION}/" -e "s/\${ACS_VERSION}/${ACS_VERSION}/" -e "s/\${RELEASE_VERSION}/${RELEASE_VERSION}/" -e "s/\${BRANCH_NAME}/${BRANCH_NAME}/" apple/release_rio_bump.patch.template | git apply
+
+# change binary compatibility reference version in root pom file
+sed -i -e 's#<japicmp.referenceVersion>.*</japicmp.referenceVersion>#<japicmp.referenceVersion>'"$BCC_RELEASE_VERSION"'</japicmp.referenceVersion>#' pom.xml
+rm -f pom.xml-e
+
+apple/update_rio_for_release.py "${RELEASE_VERSION}" "${PATCH_VERSION}.${ACS_VERSION}"
git commit -am"[apple][release] Update rio.yaml for $RELEASE_VERSION"
cd apple
diff --git a/apple/release_rio_bump.patch.template b/apple/release_rio_bump.patch.template
deleted file mode 100644
index fadf29657ff..00000000000
--- a/apple/release_rio_bump.patch.template
+++ /dev/null
@@ -1,72 +0,0 @@
-diff --git a/rio.yml b/rio.yml
-index c5d8c8e6adf..e647cf3949f 100644
---- a/rio.yml
-+++ b/rio.yml
-@@ -3,8 +3,8 @@ timeout: 120
-
- pipelines:
- # Pipeline to build Flink from Apple branch.
-- - name: publish-${BRANCH_NAME}
-- branchName: ${BRANCH_NAME}
-+ - name: publish-release-${RELEASE_VERSION}
-+ branchName: release-${RELEASE_VERSION}
- machine:
- baseImage: docker.apple.com/thomas-weise/flink-ci:latest
- env:
-@@ -23,7 +23,7 @@ pipelines:
- - pushd /workspace/apple/docker; wget https://archive.apache.org/dist/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz; popd
- package:
- # enable for release (non -SNAPSHOT) builds
-- # release: true
-+ release: true
- freeform:
- - publish:
- - repo: m2:oss-patched
-@@ -31,22 +31,20 @@ pipelines:
- - dockerfilePath: apple/docker/Dockerfile
- target: base
- context: apple/docker/
- perApplication: false
-- version: "${PATCH_VERSION}-r${RIO_BUILD_NUMBER}"
-- extraTags: [ "latest", "${PATCH_VERSION}-latest" ]
-+ version: "${PATCH_VERSION}.${ACS_VERSION}"
- publish:
- - repo: docker.apple.com/acs-flink/flink-1.${MINOR_VERSION}-acs
- - dockerfilePath: apple/docker/Dockerfile
- target: hadoop
- context: apple/docker/
- perApplication: false
-- version: "${PATCH_VERSION}-r${RIO_BUILD_NUMBER}"
-- extraTags: [ "latest", "${PATCH_VERSION}-latest" ]
-+ version: "${PATCH_VERSION}.${ACS_VERSION}"
- publish:
- - repo: docker.apple.com/acs-flink/flink-1.${MINOR_VERSION}-acs-hadoop
-
-- - name: binary-${BRANCH_NAME}
-- branchName: ${BRANCH_NAME}
-+ - name: binary-release-${RELEASE_VERSION}
-+ branchName: release-${RELEASE_VERSION}
- machine:
- baseImage: docker.apple.com/thomas-weise/flink-ci:latest
- build:
-@@ -64,19 +62,7 @@ pipelines:
- - ci stage-lib "flink-$TIMED_VERSION.tgz,snapshot/org/apache/flink/flink-dist/$VERSION/"
- package:
- # enable for release (non -SNAPSHOT) builds
-- # release: true
-+ release: true
- freeform:
- - publish:
- - repo: oss-patched-binaries-local
--
-- - name: pull-request-${BRANCH_NAME}
-- branchName: ${BRANCH_NAME}
-- timeout: 600
-- machine:
-- baseImage: docker.apple.com/thomas-weise/flink-ci:latest
-- build:
-- template: freestyle:v4:prb
-- steps:
-- - cd /workspace
-- - rm -rf .staged-artifacts
-- - mvn clean package -Phive-3.1.3.0-apple,s3-plugin -Dscala-2.12 -Dmaven.test.failure.ignore=true
diff --git a/apple/update_rio_for_release.py b/apple/update_rio_for_release.py
new file mode 100755
index 00000000000..f3aad5b5219
--- /dev/null
+++ b/apple/update_rio_for_release.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import sys
+import ruamel.yaml
+
+if len(sys.argv) != 3:
+ print(f'usage: {sys.argv[0]} release_version docker_version')
+ sys.exit(1)
+
+release_version = sys.argv[1]
+docker_version = sys.argv[2]
+
+yaml_file_name = 'rio.yml'
+yaml = ruamel.yaml.YAML()
+
+with open(yaml_file_name) as istream:
+ print(f'Loading yaml file: {yaml_file_name}\n')
+ ymldoc = yaml.load(istream)
+
+ pipelines = ymldoc['pipelines']
+
+ # Publish pipeline
+ print('Searching for publish pipeline')
+ publish_pipeline = next(filter(lambda p: p['name'].startswith('publish'), pipelines), None)
+ if publish_pipeline is None:
+ print('Publish pipeline not found')
+ sys.exit(1)
+ publish_pipeline['name'] = f'publish-release-{release_version}'
+ publish_pipeline['branchName'] = f'release-{release_version}'
+ publish_pipeline['package']['release'] = True
+ for d in publish_pipeline['package']['dockerfile']:
+ d['version'] = docker_version
+ del d['extraTags']
+ print('Publish pipeline updated\n')
+
+ # Binary pipeline
+ print('Searching for binary pipeline')
+ binary_pipeline = next(filter(lambda p: p['name'].startswith('binary'), pipelines), None)
+ if binary_pipeline is None:
+ print("Binary pipeline not found")
+ sys.exit(1)
+ binary_pipeline['name'] = f'binary-release-{release_version}'
+ binary_pipeline['branchName'] = f'release-{release_version}'
+ binary_pipeline['package']['release'] = True
+ print('Binary pipeline updated\n')
+
+ # Pull request pipeline
+ print("Searching pull request pipeline")
+ pull_request_removed = False
+ for k, v in enumerate(pipelines):
+ if v['name'].startswith('pull-request'):
+ pipelines.pop(k)
+ pull_request_removed = True
+ if pull_request_removed is False:
+ print('Pull request pipeline not found')
+ sys.exit(1)
+ print('Pull request pipeline removed\n')
+
+with open(yaml_file_name, 'w') as ostream:
+ print(f'Saving yaml file: {yaml_file_name}')
+ yaml.width = 4096
+ yaml.dump(ymldoc, ostream)
diff --git a/flink-connectors/flink-sql-connector-hive-3.1.3.0-apple/pom.xml b/flink-connectors/flink-sql-connector-hive-3.1.3.0-apple/pom.xml
index baaa8890ff5..cad70889dc6 100644
--- a/flink-connectors/flink-sql-connector-hive-3.1.3.0-apple/pom.xml
+++ b/flink-connectors/flink-sql-connector-hive-3.1.3.0-apple/pom.xml
@@ -102,6 +102,13 @@ under the License.
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>io.github.zentol.japicmp</groupId>
+ <artifactId>japicmp-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/rio.yml b/rio.yml
index 7cf48663003..54089987b10 100644
--- a/rio.yml
+++ b/rio.yml
@@ -14,7 +14,7 @@ pipelines:
steps:
- cd /workspace
- rm -rf .staged-artifacts
- - mvn clean deploy -Dscala-2.12 -Phive-3.1.3.0-apple,s3-plugin,docs-and-source -DskipTests -Dfast -DaltDeploymentRepository=local-staging::default::file://`pwd`/.staged-artifacts
+ - mvn clean deploy -Dscala-2.12 -Phive-3.1.3.0-apple,s3-plugin,docs-and-source -DskipTests -DaltDeploymentRepository=local-staging::default::file://`pwd`/.staged-artifacts
# see https://docs.aci.apple.com/rio/guide-to-rio/package-and-publish/freeform.html
# must exclude maven-metadata.xml
- ci stage-lib --exclude maven-metadata.xml --exclude *.md5 --exclude *.sha1 --many-many-artifacts ./.staged-artifacts/**