You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2022/09/19 06:18:51 UTC
[pulsar] branch master updated: [feat][CI] Add approval solution to reduce GitHub Actions resource consumption (#17693)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 1f6e7a2072e [feat][CI] Add approval solution to reduce GitHub Actions resource consumption (#17693)
1f6e7a2072e is described below
commit 1f6e7a2072e1baf4a85f78acdabd4d3a162ea934
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Mon Sep 19 09:18:42 2022 +0300
[feat][CI] Add approval solution to reduce GitHub Actions resource consumption (#17693)
Please check the proposal email on dev mailing list: https://lists.apache.org/thread/gwfmxmxlhtsjn17sxxc367jcs4pcwofz
Co-authored-by: tison <wa...@gmail.com>
---
.github/PULL_REQUEST_TEMPLATE.md | 17 ++++++++++
.github/workflows/ci-cpp-build.yaml | 8 +++--
.github/workflows/ci-go-functions.yaml | 9 +++--
.github/workflows/pulsar-ci-flaky.yaml | 7 +++-
.github/workflows/pulsar-ci.yaml | 7 +++-
build/pulsar_ci_tool.sh | 61 ++++++++++++++++++++++++++++++++++
6 files changed, 103 insertions(+), 6 deletions(-)
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index c8d670e2618..38662b9eb8e 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -76,3 +76,20 @@ This change added tests and can be verified as follows:
- [ ] `doc-complete`
(Docs have been already added)
+
+### Matching PR in forked repository
+
+PR in forked repository: <!-- ENTER URL HERE
+
+After opening this PR, the build in apache/pulsar will fail and instructions will
+be provided for opening a PR in the PR author's forked repository.
+
+apache/pulsar pull requests should be first tested in your own fork since the
+apache/pulsar CI based on GitHub Actions has constrained resources and quota.
+GitHub Actions provides separate quota for pull requests that are executed in
+a forked repository.
+
+The tests will be run in the forked repository until all PR review comments have
+been handled, the tests pass and the PR is approved by a reviewer.
+
+-->
diff --git a/.github/workflows/ci-cpp-build.yaml b/.github/workflows/ci-cpp-build.yaml
index 77cdf574396..d7d6032e600 100644
--- a/.github/workflows/ci-cpp-build.yaml
+++ b/.github/workflows/ci-cpp-build.yaml
@@ -33,12 +33,11 @@ concurrency:
jobs:
changed_files_job:
- name: 'Changed files check'
+ name: Preconditions
runs-on: ubuntu-20.04
outputs:
docs_only: ${{ needs.changed_files_job.outputs.docs_only }}
cpp_only: ${{ needs.changed_files_job.outputs.cpp_only }}
- changed_tests: ${{ steps.changes.outputs.tests_files }}
steps:
- name: checkout
uses: actions/checkout@v2
@@ -55,6 +54,11 @@ jobs:
run: |
echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
+ - name: Check if the PR has been approved for testing
+ if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+ run: |
+ build/pulsar_ci_tool.sh check_ready_to_test
+
cpp-build-centos7:
needs: changed_files_job
name: Build CPP Client on CentOS7
diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml
index da7c0a829fe..4b8b4e12947 100644
--- a/.github/workflows/ci-go-functions.yaml
+++ b/.github/workflows/ci-go-functions.yaml
@@ -36,12 +36,11 @@ env:
jobs:
changed_files_job:
- name: 'Changed files check'
+ name: Preconditions
runs-on: ubuntu-20.04
outputs:
docs_only: ${{ steps.check_changes.outputs.docs_only }}
cpp_only: ${{ steps.check_changes.outputs.cpp_only }}
- changed_tests: ${{ steps.changes.outputs.tests_files }}
steps:
- name: checkout
uses: actions/checkout@v2
@@ -57,6 +56,12 @@ jobs:
id: check_changes
run: |
echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
+ echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
+
+ - name: Check if the PR has been approved for testing
+ if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+ run: |
+ build/pulsar_ci_tool.sh check_ready_to_test
check-style:
needs: changed_files_job
diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml
index a65f6e61dec..e9150bf391d 100644
--- a/.github/workflows/pulsar-ci-flaky.yaml
+++ b/.github/workflows/pulsar-ci-flaky.yaml
@@ -37,7 +37,7 @@ env:
jobs:
changed_files_job:
- name: 'Changed files check'
+ name: Preconditions
runs-on: ubuntu-20.04
outputs:
docs_only: ${{ steps.check_changes.outputs.docs_only }}
@@ -60,6 +60,11 @@ jobs:
echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
+ - name: Check if the PR has been approved for testing
+ if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+ run: |
+ build/pulsar_ci_tool.sh check_ready_to_test
+
build-and-test:
needs: changed_files_job
name: Flaky tests suite
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index bbc60199965..74f93003f8b 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -37,7 +37,7 @@ env:
jobs:
changed_files_job:
- name: 'Changed files check'
+ name: Preconditions
runs-on: ubuntu-20.04
outputs:
docs_only: ${{ steps.check_changes.outputs.docs_only }}
@@ -60,6 +60,11 @@ jobs:
echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
+ - name: Check if the PR has been approved for testing
+ if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+ run: |
+ build/pulsar_ci_tool.sh check_ready_to_test
+
build-and-license-check:
needs: changed_files_job
name: Build and License check
diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh
index 95a23a48ee2..512163bfbf3 100755
--- a/build/pulsar_ci_tool.sh
+++ b/build/pulsar_ci_tool.sh
@@ -154,6 +154,67 @@ function ci_move_test_reports() {
)
}
+function ci_check_ready_to_test() {
+ if [[ -z "$GITHUB_EVENT_PATH" ]]; then
+ >&2 echo "GITHUB_EVENT_PATH isn't set"
+ return 1
+ fi
+ # check ready-to-test label
+ if jq -e '.pull_request.labels[] | .name | select(. == "ready-to-test")' "$GITHUB_EVENT_PATH" &> /dev/null; then
+ return 0
+ fi
+
+ # check if the PR has been approved
+ PR_NUM=$(jq -r '.pull_request.number' "${GITHUB_EVENT_PATH}")
+ REPO_FULL_NAME=$(jq -r '.repository.full_name' "${GITHUB_EVENT_PATH}")
+ REPO_NAME=$(basename "${REPO_FULL_NAME}")
+ REPO_OWNER=$(dirname "${REPO_FULL_NAME}")
+ # use graphql query to find out reviewDecision
+ PR_REVIEW_DECISION=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" -X POST -d '{"query": "query { repository(name: \"'${REPO_NAME}'\", owner: \"'${REPO_OWNER}'\") { pullRequest(number: '${PR_NUM}') { reviewDecision } } }"}' https://api.github.com/graphql |jq -r '.data.repository.pullRequest.reviewDecision')
+ echo "Review decision for PR #${PR_NUM} in repository ${REPO_OWNER}/${REPO_NAME} is ${PR_REVIEW_DECISION}"
+ if [[ "$PR_REVIEW_DECISION" == "APPROVED" ]]; then
+ return 0
+ fi
+
+ FORK_REPO_URL=$(jq -r '.pull_request.head.repo.html_url' "$GITHUB_EVENT_PATH")
+ PR_BRANCH_LABEL=$(jq -r '.pull_request.head.label' "$GITHUB_EVENT_PATH")
+ PR_URL=$(jq -r '.pull_request.html_url' "$GITHUB_EVENT_PATH")
+ FORK_PR_TITLE_URL_ENCODED=$(jq -r '"[run-tests] " + .pull_request.title | @uri' "$GITHUB_EVENT_PATH")
+ FORK_PR_BODY_URL_ENCODED=$(jq -n -r "\"This PR is for running tests for upstream PR ${PR_URL}.\n\n<!-- Before creating this PR, please ensure that the fork $FORK_REPO_URL is up to date with https://github.com/apache/pulsar -->\" | @uri")
+ >&2 tee -a "$GITHUB_STEP_SUMMARY" <<EOF
+
+# Instructions for proceeding with the pull request:
+
+apache/pulsar pull requests should be first tested in your own fork since the apache/pulsar CI based on
+GitHub Actions has constrained resources and quota. GitHub Actions provides separate quota for
+pull requests that are executed in a forked repository.
+
+1. Go to ${FORK_REPO_URL} and ensure that your branch is up to date with https://github.com/apache/pulsar
+ Sync your fork if it's behind.
+2. Open a pull request to your own fork. You can use this link to create the pull request in
+ your own fork:
+ [Create PR in fork for running tests](${FORK_REPO_URL}/compare/master...${PR_BRANCH_LABEL}?expand=1&title=${FORK_PR_TITLE_URL_ENCODED}&body=${FORK_PR_BODY_URL_ENCODED})
+3. Edit the description of the pull request ${PR_URL} and add the link to the PR that you opened to your own fork
+ so that the reviewer can verify that tests pass in your own fork.
+4. Ensure that tests pass in your own fork. Your own fork will be used to run the tests during the PR review
+ and any changes made during the review. You as a PR author are responsible for following up on test failures.
+ Please report any flaky tests as new issues at https://github.com/apache/pulsar/issues
+ after checking that the flaky test isn't already reported.
+5. When the PR is approved, it will be possible to restart the Pulsar CI workflow within apache/pulsar
+ repository by adding a comment "/pulsarbot rerun-failure-checks" to the PR.
+ An alternative for the PR approval is to add a ready-to-test label to the PR. This can be done
+ by Apache Pulsar committers.
+6. When tests pass on the apache/pulsar side, the PR can be merged by a Apache Pulsar Committer.
+
+If you have any trouble you can get support in multiple ways:
+* by sending email to the [dev mailing list](mailto:dev@pulsar.apache.org) ([subscribe](mailto:dev-subscribe@pulsar.apache.org))
+* on the [#contributors channel on Pulsar Slack](https://apache-pulsar.slack.com/channels/contributors) ([join](https://pulsar.apache.org/community#section-discussions))
+* in apache/pulsar [GitHub discussions Q&A](https://github.com/apache/pulsar/discussions/categories/q-a)
+
+EOF
+ return 1
+}
+
if [ -z "$1" ]; then
echo "usage: $0 [ci_tool_function_name]"
echo "Available ci tool functions:"