You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2023/03/23 12:57:08 UTC
[spark] branch master updated: [SPARK-41537][INFRA][TESTS] Github Workflow Check for Breaking Changes in Spark Connect Proto
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 280bc3376ae [SPARK-41537][INFRA][TESTS] Github Workflow Check for Breaking Changes in Spark Connect Proto
280bc3376ae is described below
commit 280bc3376ae9897e2c870ee64571ecf38ce4acee
Author: Martin Grund <ma...@databricks.com>
AuthorDate: Thu Mar 23 21:55:44 2023 +0900
[SPARK-41537][INFRA][TESTS] Github Workflow Check for Breaking Changes in Spark Connect Proto
### What changes were proposed in this pull request?
This PR adds a new script to check for breaking changes in the Spark Connect protos using the `buf` breaking changes detector.
Please see here for more details: https://docs.buf.build/tour/detect-breaking-changes
### Why are the changes needed?
Testing / CI / CD
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Manual WF testing.
Closes #39294 from grundprinzip/SPARK-41537.
Lead-authored-by: Martin Grund <ma...@databricks.com>
Co-authored-by: Martin Grund <gr...@gmail.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.github/workflows/build_and_test.yml | 75 ++++++++++++++++++++++++++++++++++++
dev/detect_breaking_changes.sh | 36 +++++++++++++++++
2 files changed, 111 insertions(+)
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index ebc40e92791..56e8ac87ebe 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -101,6 +101,7 @@ jobs:
\"java-11-17\": \"true\",
\"lint\" : \"true\",
\"k8s-integration-tests\" : \"true\",
+ \"proto-breaking-changes-check\" : \"true\",
}"
echo $precondition # For debugging
# Remove `\n` to avoid "Invalid format" error
@@ -493,6 +494,80 @@ jobs:
name: test-results-sparkr--8-${{ inputs.hadoop }}-hive2.3
path: "**/target/test-reports/*.xml"
+ proto-breaking-changes-check-master:
+ needs: [precondition]
+ if: always() && fromJson(needs.precondition.outputs.required).proto-breaking-changes-check == 'true'
+ name: Spark Connect proto backwards compatibility check (master)
+ continue-on-error: true
+ runs-on: ubuntu-22.04
+ env:
+ LC_ALL: C.UTF-8
+ LANG: C.UTF-8
+ PYSPARK_DRIVER_PYTHON: python3.9
+ PYSPARK_PYTHON: python3.9
+ steps:
+ - name: Checkout Spark repository
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+ repository: apache/spark
+ ref: ${{ inputs.branch }}
+ - name: Add GITHUB_WORKSPACE to git trust safe.directory
+ run: |
+ git config --global --add safe.directory ${GITHUB_WORKSPACE}
+ - name: Sync the current branch with the latest in Apache Spark
+ if: github.repository != 'apache/spark'
+ run: |
+ git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/}
+ git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
+ git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty
+ # Install the `buf` CLI
+ - uses: bufbuild/buf-setup-action@v1
+ with:
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ # Run buf breaking.
+ - uses: bufbuild/buf-breaking-action@v1
+ with:
+ input: connector/connect/common/src/main
+ against: 'https://github.com/apache/spark.git#branch=master,subdir=connector/connect/common/src/main'
+
+ proto-breaking-changes-check-release:
+ needs: [precondition]
+ if: always() && fromJson(needs.precondition.outputs.required).proto-breaking-changes-check == 'true'
+ name: Spark Connect proto backwards compatibility check (master)
+ runs-on: ubuntu-22.04
+ env:
+ LC_ALL: C.UTF-8
+ LANG: C.UTF-8
+ PYSPARK_DRIVER_PYTHON: python3.9
+ PYSPARK_PYTHON: python3.9
+ steps:
+ - name: Checkout Spark repository
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+ repository: apache/spark
+ ref: ${{ inputs.branch }}
+ - name: Add GITHUB_WORKSPACE to git trust safe.directory
+ run: |
+ git config --global --add safe.directory ${GITHUB_WORKSPACE}
+ - name: Sync the current branch with the latest in Apache Spark
+ if: github.repository != 'apache/spark'
+ run: |
+ git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/}
+ git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
+ git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty
+ # Install the `buf` CLI
+ - uses: bufbuild/buf-setup-action@v1
+ with:
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ # Run buf breaking.
+ - uses: bufbuild/buf-breaking-action@v1
+ with:
+ input: connector/connect/common/src/main
+ against: 'https://github.com/apache/spark.git#branch=branch-3.4,subdir=connector/connect/common/src/main'
+
+
# Static analysis, and documentation build
lint:
needs: [precondition, infra-image]
diff --git a/dev/detect_breaking_changes.sh b/dev/detect_breaking_changes.sh
new file mode 100755
index 00000000000..709af0ed074
--- /dev/null
+++ b/dev/detect_breaking_changes.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+if [[ $# -gt 0 ]]; then
+ echo "Illegal number of parameters."
+ echo "Usage: ./connector/connect/dev/detect_breaking_changes.sh"
+ exit -1
+fi
+
+
+SPARK_HOME="$(cd "`dirname $0`"/../../..; pwd)"
+cd "$SPARK_HOME"
+
+pushd connector/connect/common/src/main
+
+buf breaking --against "https://github.com/apache/spark/archive/master.zip#strip_components=1,subdir=connector/connect/common/src/main"
+if [[ $? -ne -0 ]]; then
+ echo "Buf detected breaking changes for your Pull Request. Please verify."
+ echo "Please make sure your branch is current against spark/master."
+ exit 1
+fi
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org