You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2022/09/21 11:40:52 UTC
[pulsar] branch branch-2.11 updated: [fix][CI] Synchronize GitHub Actions from master branch
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-2.11
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/branch-2.11 by this push:
new eda5e89e256 [fix][CI] Synchronize GitHub Actions from master branch
eda5e89e256 is described below
commit eda5e89e25615150b37c17364e87c1f5ee007883
Author: Lari Hotari <lh...@apache.org>
AuthorDate: Wed Sep 21 14:38:37 2022 +0300
[fix][CI] Synchronize GitHub Actions from master branch
- omit PLUGIN group since it's not part of branch-2.11
---
.github/changes-filter.yaml | 5 +
.github/workflows/ci-cpp-build.yaml | 25 +-
.github/workflows/ci-documentbot.yml | 19 +-
.github/workflows/ci-go-functions.yaml | 14 +-
.github/workflows/ci-maven-cache-update.yaml | 6 +-
.github/workflows/ci-owasp-dependency-check.yaml | 4 +-
.github/workflows/ci-pulsarbot.yaml | 2 +-
.github/workflows/ci-stale-issue-pr.yaml | 2 +-
.github/workflows/pulsar-ci-flaky.yaml | 12 +-
.github/workflows/pulsar-ci.yaml | 316 +++++++++++++++++++----
build/pulsar_ci_tool.sh | 91 +++++++
build/run_unit_group.sh | 2 +-
12 files changed, 408 insertions(+), 90 deletions(-)
diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml
index b14153acb82..82f5b22a8aa 100644
--- a/.github/changes-filter.yaml
+++ b/.github/changes-filter.yaml
@@ -12,3 +12,8 @@ tests:
- added|modified: '**/src/test/java/**/*.java'
cpp:
- 'pulsar-client-cpp/**'
+need_owasp:
+ - 'pom.xml'
+ - '**/pom.xml'
+ - 'src/owasp-dependency-check-false-positives.xml'
+ - 'src/owasp-dependency-check-suppressions.xml'
\ No newline at end of file
diff --git a/.github/workflows/ci-cpp-build.yaml b/.github/workflows/ci-cpp-build.yaml
index 9c78053bcad..7a450efe972 100644
--- a/.github/workflows/ci-cpp-build.yaml
+++ b/.github/workflows/ci-cpp-build.yaml
@@ -33,12 +33,11 @@ concurrency:
jobs:
changed_files_job:
- name: 'Changed files check'
+ name: Preconditions
runs-on: ubuntu-20.04
outputs:
docs_only: ${{ needs.changed_files_job.outputs.docs_only }}
cpp_only: ${{ needs.changed_files_job.outputs.cpp_only }}
- changed_tests: ${{ steps.changes.outputs.tests_files }}
steps:
- name: checkout
uses: actions/checkout@v2
@@ -55,10 +54,18 @@ jobs:
run: |
echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
+ - name: Check if the PR has been approved for testing
+ if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+ env:
+ GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ build/pulsar_ci_tool.sh check_ready_to_test
+
cpp-build-centos7:
needs: changed_files_job
- name:
- runs-on: ubuntu-latest
+ name: Build CPP Client on CentOS7
+ runs-on: ubuntu-20.04
if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
timeout-minutes: 120
@@ -77,7 +84,7 @@ jobs:
cpp-build-windows:
needs: changed_files_job
timeout-minutes: 120
- name: ${{ matrix.name }}
+ name: Build CPP Client on ${{ matrix.name }}
runs-on: ${{ matrix.os }}
if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
env:
@@ -184,8 +191,8 @@ jobs:
fi
cpp-deb-rpm-packaging:
needs: changed_files_job
- name:
- runs-on: ubuntu-latest
+ name: Build CPP Client on RPM
+ runs-on: ubuntu-20.04
timeout-minutes: 120
if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
@@ -233,8 +240,8 @@ jobs:
build-python-wheel:
needs: changed_files_job
- name:
- runs-on: ubuntu-latest
+ name: Build Python Client
+ runs-on: ubuntu-20.04
timeout-minutes: 120
if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
diff --git a/.github/workflows/ci-documentbot.yml b/.github/workflows/ci-documentbot.yml
index 8c31223fdef..e859940080f 100644
--- a/.github/workflows/ci-documentbot.yml
+++ b/.github/workflows/ci-documentbot.yml
@@ -28,29 +28,18 @@ on:
- unlabeled
concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
+ group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.number }}
cancel-in-progress: true
jobs:
label:
- if: ${{ github.repository == 'apache/pulsar' }}
+ if: (github.repository == 'apache/pulsar') && (github.event.pull_request.state == 'open')
permissions:
pull-requests: write
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
steps:
- - name: Checkout action
- uses: actions/checkout@v3
- with:
- repository: apache/pulsar-test-infra
- ref: master
-
- - name: Set up Go
- uses: actions/setup-go@v3
- with:
- go-version: 1.18
-
- name: Labeling
- uses: ./docbot
+ uses: apache/pulsar-test-infra/docbot@master
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
LABEL_WATCH_LIST: 'doc,doc-required,doc-not-needed,doc-complete'
diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml
index 44edd39469e..266bfd57723 100644
--- a/.github/workflows/ci-go-functions.yaml
+++ b/.github/workflows/ci-go-functions.yaml
@@ -36,12 +36,11 @@ env:
jobs:
changed_files_job:
- name: 'Changed files check'
+ name: Preconditions
runs-on: ubuntu-20.04
outputs:
docs_only: ${{ steps.check_changes.outputs.docs_only }}
cpp_only: ${{ steps.check_changes.outputs.cpp_only }}
- changed_tests: ${{ steps.changes.outputs.tests_files }}
steps:
- name: checkout
uses: actions/checkout@v2
@@ -57,12 +56,21 @@ jobs:
id: check_changes
run: |
echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
+ echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
+
+ - name: Check if the PR has been approved for testing
+ if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+ env:
+ GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ build/pulsar_ci_tool.sh check_ready_to_test
check-style:
needs: changed_files_job
if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
name: Go ${{ matrix.go-version }} Functions style check
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
strategy:
matrix:
go-version: [1.15, 1.16, 1.17]
diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml
index d78f77707fd..b432541bce1 100644
--- a/.github/workflows/ci-maven-cache-update.yaml
+++ b/.github/workflows/ci-maven-cache-update.yaml
@@ -55,16 +55,16 @@ jobs:
matrix:
include:
- name: all modules
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
cache_name: 'm2-dependencies-all'
mvn_arguments: ''
- name: all modules - macos
- runs-on: macos-latest
+ runs-on: macos-11
cache_name: 'm2-dependencies-all'
- name: core-modules
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
cache_name: 'm2-dependencies-core-modules'
mvn_arguments: '-Pcore-modules,-main'
diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml
index a9e114c21f9..35c8e6ebebd 100644
--- a/.github/workflows/ci-owasp-dependency-check.yaml
+++ b/.github/workflows/ci-owasp-dependency-check.yaml
@@ -29,7 +29,7 @@ jobs:
run-owasp-dependency-check:
if: ${{ github.repository == 'apache/pulsar' }}
name: Run OWASP Dependency Check
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 45
strategy:
fail-fast: false
@@ -37,6 +37,8 @@ jobs:
include:
- name: master
checkout_branch: 'master'
+ - name: branch-2.11
+ checkout_branch: 'branch-2.11'
- name: branch-2.10
checkout_branch: 'branch-2.10'
- name: branch-2.9
diff --git a/.github/workflows/ci-pulsarbot.yaml b/.github/workflows/ci-pulsarbot.yaml
index d6be27027cb..157d668e6cd 100644
--- a/.github/workflows/ci-pulsarbot.yaml
+++ b/.github/workflows/ci-pulsarbot.yaml
@@ -24,7 +24,7 @@ on:
jobs:
pulsarbot:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
timeout-minutes: 10
if: github.event_name == 'issue_comment' && contains(github.event.comment.body, '/pulsarbot')
steps:
diff --git a/.github/workflows/ci-stale-issue-pr.yaml b/.github/workflows/ci-stale-issue-pr.yaml
index cf1d30a4730..48ed5246001 100644
--- a/.github/workflows/ci-stale-issue-pr.yaml
+++ b/.github/workflows/ci-stale-issue-pr.yaml
@@ -5,7 +5,7 @@ on:
jobs:
stale:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
steps:
- uses: actions/stale@v4
with:
diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml
index 67f2f0cfb67..03e3adff33a 100644
--- a/.github/workflows/pulsar-ci-flaky.yaml
+++ b/.github/workflows/pulsar-ci-flaky.yaml
@@ -37,7 +37,7 @@ env:
jobs:
changed_files_job:
- name: 'Changed files check'
+ name: Preconditions
runs-on: ubuntu-20.04
outputs:
docs_only: ${{ steps.check_changes.outputs.docs_only }}
@@ -60,6 +60,14 @@ jobs:
echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
+ - name: Check if the PR has been approved for testing
+ if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+ env:
+ GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ build/pulsar_ci_tool.sh check_ready_to_test
+
build-and-test:
needs: changed_files_job
name: Flaky tests suite
@@ -75,7 +83,7 @@ jobs:
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
- if: ${{ github.repository != 'apache/pulsar' }}
+ if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
with:
limit-access-to-actor: true
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index aa3959f6cfa..0436d9f4c3d 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -37,7 +37,7 @@ env:
jobs:
changed_files_job:
- name: 'Changed files check'
+ name: Preconditions
runs-on: ubuntu-20.04
outputs:
docs_only: ${{ steps.check_changes.outputs.docs_only }}
@@ -60,6 +60,14 @@ jobs:
echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
+ - name: Check if the PR has been approved for testing
+ if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+ env:
+ GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ build/pulsar_ci_tool.sh check_ready_to_test
+
build-and-license-check:
needs: changed_files_job
name: Build and License check
@@ -75,7 +83,7 @@ jobs:
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
- if: ${{ github.repository != 'apache/pulsar' }}
+ if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
with:
limit-access-to-actor: true
@@ -115,47 +123,18 @@ jobs:
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \
tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \
-I zstd -cf - .m2/repository/org/apache/pulsar
+ cd $GITHUB_WORKSPACE
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-server-distribution \
+ tar -I zstd -cf - distribution/server/target/apache-pulsar-*-bin.tar.gz
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
- if: ${{ failure() && github.repository != 'apache/pulsar' }}
+ if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
- macos-build:
- name:
- runs-on: macos-latest
- timeout-minutes: 120
- needs: [ 'changed_files_job', 'build-and-license-check' ]
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Cache Maven dependencies
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-all-
-
- - name: Set up JDK 17
- uses: actions/setup-java@v2
- with:
- distribution: 'temurin'
- java-version: 17
-
- - name: build package
- run: mvn -B clean package -DskipTests -T 1C -ntp
unit-tests:
name: CI - Unit - ${{ matrix.name }}
@@ -194,7 +173,7 @@ jobs:
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
- if: ${{ github.repository != 'apache/pulsar' }}
+ if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
with:
limit-access-to-actor: true
@@ -266,7 +245,7 @@ jobs:
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
- if: ${{ failure() && github.repository != 'apache/pulsar' }}
+ if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
@@ -288,6 +267,7 @@ jobs:
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
+ if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
with:
limit-access-to-actor: true
@@ -320,7 +300,7 @@ jobs:
run: |
# build docker image
mvn -B -am -pl tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
- -Dmaven.test.skip=true -Ddockerfile.build.squash=true -DskipSourceReleaseAssembly=true \
+ -Dmaven.test.skip=true -Ddocker.squash=true -DskipSourceReleaseAssembly=true \
-Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
- name: save docker image apachepulsar/java-test-image:latest to Github artifact cache
@@ -330,7 +310,7 @@ jobs:
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
- if: ${{ failure() && github.repository != 'apache/pulsar' }}
+ if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
@@ -385,7 +365,7 @@ jobs:
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
- if: ${{ github.repository != 'apache/pulsar' }}
+ if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
with:
limit-access-to-actor: true
@@ -469,7 +449,7 @@ jobs:
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
- if: ${{ failure() && github.repository != 'apache/pulsar' }}
+ if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
@@ -497,6 +477,94 @@ jobs:
run: |
gh-actions-artifact-client.js delete pulsar-java-test-image.zst
+ cpp-tests:
+ name: CI - CPP, Python Tests
+ runs-on: ubuntu-20.04
+ timeout-minutes: 120
+ needs: [
+ 'changed_files_job',
+ 'integration-tests'
+ ]
+ if: always()
+ steps:
+ - name: check condition
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ if [[ ! ( ( \
+ "${{needs.changed_files_job.outputs.cpp_only}}" == "false" \
+ && "${{ needs.integration-tests.result }}" == "success" \
+ ) || ( \
+ "${{needs.changed_files_job.outputs.cpp_only}}" == "true" \
+ ) ) ]]; then
+ echo "Required jobs haven't been completed successfully."
+ exit 1
+ fi
+
+ - name: checkout
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/tune-runner-vm
+
+ - name: Cache local Maven repository
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/pulsar
+ key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-m2-dependencies-core-modules-
+
+ - name: Set up JDK 17
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/setup-java@v2
+ with:
+ distribution: 'temurin'
+ java-version: 17
+
+ - name: Clean Disk
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/clean-disk
+
+ - name: Install gh-actions-artifact-client.js
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
+
+ - name: Restore maven build results from Github artifact cache
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ cd $HOME
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
+ cd $GITHUB_WORKSPACE
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-server-distribution
+
+ - name: copy python tests
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ mvn -B -Pskip-all -ntp -pl pulsar-functions/instance package
+
+ - name: build cpp artifacts
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ echo "Build C++ client library"
+ pulsar-client-cpp/docker-build.sh
+
+ - name: run c++ tests
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: pulsar-client-cpp/docker-tests.sh
+
+ - name: Upload test-logs
+ if: ${{ failure() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/upload-artifact@v3
+ continue-on-error: true
+ with:
+ name: cpp-tests-logs
+ path: test-logs
+
pulsar-test-latest-version-image:
name: Build Pulsar docker image
runs-on: ubuntu-20.04
@@ -514,7 +582,7 @@ jobs:
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
- if: ${{ github.repository != 'apache/pulsar' }}
+ if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
with:
limit-access-to-actor: true
@@ -552,9 +620,33 @@ jobs:
# build docker image
# include building of Pulsar SQL, Connectors, Offloaders and server distros
mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,tests/docker-images/latest-version-image install \
- -Pmain,docker -Dmaven.test.skip=true -Ddockerfile.build.squash=true \
+ -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \
-Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
+ - name: Clean up disk space
+ run: |
+ # release disk space since saving docker image consumes local disk space
+ #
+ echo "::group::Available diskspace before cleaning"
+ time df -BM / /mnt
+ echo "::endgroup::"
+ echo "::group::Clean build directory"
+ # docker build changes some files to root ownership, fix this before deleting files
+ sudo chown -R $USER:$GROUP .
+ # clean build directories
+ time git clean -fdx
+ echo "::endgroup::"
+ echo "::group::Available diskspace after cleaning build directory"
+ time df -BM / /mnt
+ echo "::endgroup::"
+ echo "::group::Delete maven repository"
+ # delete maven repository
+ time rm -rf ~/.m2/repository
+ echo "::endgroup::"
+ echo "::group::Available diskspace after cleaning maven repository"
+ time df -BM / /mnt
+ echo "::endgroup::"
+
- name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image
@@ -562,7 +654,7 @@ jobs:
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
- if: ${{ failure() && github.repository != 'apache/pulsar' }}
+ if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
@@ -612,7 +704,7 @@ jobs:
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
- if: ${{ github.repository != 'apache/pulsar' }}
+ if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
with:
limit-access-to-actor: true
@@ -690,7 +782,7 @@ jobs:
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
- if: ${{ failure() && github.repository != 'apache/pulsar' }}
+ if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
@@ -707,9 +799,6 @@ jobs:
fail-fast: false
matrix:
include:
- - name: Plugin
- group: PLUGIN
-
- name: Pulsar IO - Oracle
group: PULSAR_IO_ORA
@@ -723,7 +812,7 @@ jobs:
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
- if: ${{ github.repository != 'apache/pulsar' }}
+ if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
with:
limit-access-to-actor: true
@@ -801,7 +890,7 @@ jobs:
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
- if: ${{ failure() && github.repository != 'apache/pulsar' }}
+ if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
@@ -830,11 +919,109 @@ jobs:
run: |
gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst
+ macos-build:
+ name: Build Pulsar on MacOS
+ runs-on: macos-11
+ timeout-minutes: 120
+ needs: ['changed_files_job', 'integration-tests']
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ steps:
+ - name: checkout
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ uses: ./.github/actions/tune-runner-vm
+
+ - name: Cache Maven dependencies
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/pulsar
+ key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-m2-dependencies-all-
+
+ - name: Set up JDK 17
+ uses: actions/setup-java@v2
+ with:
+ distribution: 'temurin'
+ java-version: 17
+
+ - name: build package
+ run: mvn -B clean package -DskipTests -T 1C -ntp
+
+ owasp-dep-check:
+ name: OWASP dependency check
+ runs-on: ubuntu-20.04
+ timeout-minutes: 120
+ needs: [ 'changed_files_job', 'integration-tests' ]
+ if: ${{ needs.changed_files_job.outputs.need_owasp == 'true' }}
+ steps:
+ - name: checkout
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ uses: ./.github/actions/tune-runner-vm
+
+ - name: Setup ssh access to build runner VM
+ # ssh access is enabled for builds in own forks
+ if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
+ uses: ./.github/actions/ssh-access
+ with:
+ limit-access-to-actor: true
+
+ - name: Cache Maven dependencies
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/pulsar
+ key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-m2-dependencies-core-modules-
+ - name: Set up JDK ${{ matrix.jdk || '17' }}
+ uses: actions/setup-java@v2
+ with:
+ distribution: 'temurin'
+ java-version: ${{ matrix.jdk || '17' }}
+
+ - name: Clean Disk
+ uses: ./.github/actions/clean-disk
+
+ - name: Install gh-actions-artifact-client.js
+ uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
+
+ - name: Restore maven build results from Github artifact cache
+ run: |
+ cd $HOME
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
+ # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan.
+ - name: run "clean verify" to trigger dependency check
+ run: mvn -q -B -ntp verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'
+
+ - name: Upload report
+ uses: actions/upload-artifact@v3
+ if: ${{ cancelled() || failure() }}
+ continue-on-error: true
+ with:
+ name: dependency report
+ path: target/dependency-check-report.html
+
+ - name: Wait for ssh connection when build fails
+ # ssh access is enabled for builds in own forks
+ uses: ./.github/actions/ssh-access
+ if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
+ continue-on-error: true
+ with:
+ action: wait
+
# This job is required for pulls to be merged.
# It depends on all other jobs in this workflow.
# It cleans up the binaries in the same job in order to not spin up another runner for basically doing nothing.
pulsar-ci-checks-completed:
name: "Pulsar CI checks completed"
+ if: always()
runs-on: ubuntu-20.04
timeout-minutes: 10
needs: [
@@ -842,10 +1029,30 @@ jobs:
'unit-tests',
'integration-tests',
'system-tests',
- 'macos-build'
+ 'flaky-system-tests',
+ 'macos-build',
+ 'cpp-tests'
]
- if: always()
steps:
+ - name: Check that all required jobs were completed successfully
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ if [[ ! ( ( \
+ "${{needs.changed_files_job.outputs.cpp_only}}" == "false" \
+ && "${{ needs.unit-tests.result }}" == "success" \
+ && "${{ needs.integration-tests.result }}" == "success" \
+ && "${{ needs.system-tests.result }}" == "success" \
+ && "${{ needs.macos-build.result }}" == "success" \
+ && "${{ needs.cpp-tests.result }}" == "success" \
+ ) || ( \
+ "${{needs.changed_files_job.outputs.cpp_only}}" == "true" \
+ && "${{ needs.system-tests.result }}" == "success" \
+ && "${{ needs.cpp-tests.result }}" == "success" \
+ ) ) ]]; then
+ echo "Required jobs haven't been completed successfully."
+ exit 1
+ fi
+
- name: checkout
if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: actions/checkout@v2
@@ -861,4 +1068,5 @@ jobs:
- name: Delete maven repository binaries from GitHub Actions Artifacts
if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
run: |
- gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst
\ No newline at end of file
+ gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true
+ gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true
diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh
index 95a23a48ee2..5c5a294c17d 100755
--- a/build/pulsar_ci_tool.sh
+++ b/build/pulsar_ci_tool.sh
@@ -154,6 +154,97 @@ function ci_move_test_reports() {
)
}
+function ci_check_ready_to_test() {
+ if [[ -z "$GITHUB_EVENT_PATH" ]]; then
+ >&2 echo "GITHUB_EVENT_PATH isn't set"
+ return 1
+ fi
+
+ PR_JSON_URL=$(jq -r '.pull_request.url' "${GITHUB_EVENT_PATH}")
+ echo "Refreshing $PR_JSON_URL..."
+ PR_JSON=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "${PR_JSON_URL}")
+
+ if printf "%s" "${PR_JSON}" | jq -e '.draft | select(. == true)' &> /dev/null; then
+ echo "PR is draft."
+ elif ! ( printf "%s" "${PR_JSON}" | jq -e '.mergeable | select(. == true)' &> /dev/null ); then
+ echo "PR isn't mergeable."
+ else
+ # check ready-to-test label
+ if printf "%s" "${PR_JSON}" | jq -e '.labels[] | .name | select(. == "ready-to-test")' &> /dev/null; then
+ echo "Found ready-to-test label."
+ return 0
+ else
+ echo "There is no ready-to-test label on the PR."
+ fi
+
+ # check if the PR has been approved
+ PR_NUM=$(jq -r '.pull_request.number' "${GITHUB_EVENT_PATH}")
+ REPO_FULL_NAME=$(jq -r '.repository.full_name' "${GITHUB_EVENT_PATH}")
+ REPO_NAME=$(basename "${REPO_FULL_NAME}")
+ REPO_OWNER=$(dirname "${REPO_FULL_NAME}")
+ # use graphql query to find out reviewDecision
+ PR_REVIEW_DECISION=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" -X POST -d '{"query": "query { repository(name: \"'${REPO_NAME}'\", owner: \"'${REPO_OWNER}'\") { pullRequest(number: '${PR_NUM}') { reviewDecision } } }"}' https://api.github.com/graphql |jq -r '.data.repository.pullRequest.reviewDecision')
+ echo "Review decision for PR #${PR_NUM} in repository ${REPO_OWNER}/${REPO_NAME} is ${PR_REVIEW_DECISION}"
+ if [[ "$PR_REVIEW_DECISION" == "APPROVED" ]]; then
+ return 0
+ fi
+ fi
+
+ FORK_REPO_URL=$(jq -r '.pull_request.head.repo.html_url' "$GITHUB_EVENT_PATH")
+ PR_BRANCH_LABEL=$(jq -r '.pull_request.head.label' "$GITHUB_EVENT_PATH")
+ PR_BASE_BRANCH=$(jq -r '.pull_request.base.ref' "$GITHUB_EVENT_PATH")
+ PR_URL=$(jq -r '.pull_request.html_url' "$GITHUB_EVENT_PATH")
+ FORK_PR_TITLE_URL_ENCODED=$(printf "%s" "${PR_JSON}" | jq -r '"[run-tests] " + .title | @uri')
+ FORK_PR_BODY_URL_ENCODED=$(jq -n -r "\"This PR is for running tests for upstream PR ${PR_URL}.\n\n<!-- Before creating this PR, please ensure that the fork $FORK_REPO_URL is up to date with https://github.com/apache/pulsar -->\" | @uri")
+ if [[ "$PR_BASE_BRANCH" != "master" ]]; then
+ sync_non_master_fork_docs=$(cat <<EOF
+ \\$('\n')
+ If ${FORK_REPO_URL}/tree/${PR_BASE_BRANCH} is missing, you must sync the branch ${PR_BASE_BRANCH} on the command line.
+ \`\`\`
+ git fetch https://github.com/apache/pulsar ${PR_BASE_BRANCH}
+ git push ${FORK_REPO_URL} FETCH_HEAD:${PR_BASE_BRANCH}
+ \`\`\`
+EOF
+)
+ else
+ sync_non_master_fork_docs=""
+ fi
+
+ >&2 tee -a "$GITHUB_STEP_SUMMARY" <<EOF
+
+# Instructions for proceeding with the pull request:
+
+apache/pulsar pull requests should be first tested in your own fork since the apache/pulsar CI based on
+GitHub Actions has constrained resources and quota. GitHub Actions provides separate quota for
+pull requests that are executed in a forked repository.
+
+1. Go to ${FORK_REPO_URL}/tree/${PR_BASE_BRANCH} and ensure that your ${PR_BASE_BRANCH} branch is up to date
+ with https://github.com/apache/pulsar \\
+ [Sync your fork if it's behind.](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork)${sync_non_master_fork_docs}
+2. Open a pull request to your own fork. You can use this link to create the pull request in
+ your own fork:
+ [Create PR in fork for running tests](${FORK_REPO_URL}/compare/${PR_BASE_BRANCH}...${PR_BRANCH_LABEL}?expand=1&title=${FORK_PR_TITLE_URL_ENCODED}&body=${FORK_PR_BODY_URL_ENCODED})
+3. Edit the description of the pull request ${PR_URL} and add the link to the PR that you opened to your own fork
+ so that the reviewer can verify that tests pass in your own fork.
+4. Ensure that tests pass in your own fork. Your own fork will be used to run the tests during the PR review
+ and any changes made during the review. You as a PR author are responsible for following up on test failures.
+ Please report any flaky tests as new issues at https://github.com/apache/pulsar/issues
+ after checking that the flaky test isn't already reported.
+5. When the PR is approved, it will be possible to restart the Pulsar CI workflow within apache/pulsar
+ repository by adding a comment "/pulsarbot rerun-failure-checks" to the PR.
+ An alternative for the PR approval is to add a ready-to-test label to the PR. This can be done
+ by Apache Pulsar committers.
+6. When tests pass on the apache/pulsar side, the PR can be merged by a Apache Pulsar Committer.
+
+If you have any trouble you can get support in multiple ways:
+* by sending email to the [dev mailing list](mailto:dev@pulsar.apache.org) ([subscribe](mailto:dev-subscribe@pulsar.apache.org))
+* on the [#contributors channel on Pulsar Slack](https://apache-pulsar.slack.com/channels/contributors) ([join](https://pulsar.apache.org/community#section-discussions))
+* in apache/pulsar [GitHub discussions Q&A](https://github.com/apache/pulsar/discussions/categories/q-a)
+
+EOF
+ return 1
+}
+
if [ -z "$1" ]; then
echo "usage: $0 [ci_tool_function_name]"
echo "Available ci tool functions:"
diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh
index f011fe6bfc8..87836261609 100755
--- a/build/run_unit_group.sh
+++ b/build/run_unit_group.sh
@@ -163,7 +163,7 @@ function test_group_pulsar_io() {
echo "::endgroup::"
echo "::group::Running pulsar-sql tests"
- mvn_test --install -Ppulsar-sql-tests,-main
+ mvn_test --install -Ppulsar-sql-tests,-main -DtestForkCount=1
echo "::endgroup::"
}