You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by ni...@apache.org on 2022/09/09 07:53:25 UTC
[pulsar] 06/08: [ci] Reduce runners load for pulls that affect only doc or cpp changes (#17541)
This is an automated email from the ASF dual-hosted git repository.
nicoloboschi pushed a commit to branch branch-2.11
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit c615a188964e61497b5e97f314387ebf32fd6c14
Author: Nicolò Boschi <bo...@gmail.com>
AuthorDate: Thu Sep 8 17:03:43 2022 +0200
[ci] Reduce runners load for pulls that affect only doc or cpp changes (#17541)
* [ci] Reduce runners load for pulls that affects only doc or cpp changes
* add final jobs to mark required workflows as completed even if they are skipped
* test parametrized job id
* fix latest step
(cherry picked from commit 7e869682a62088a31168b3de1f8c3cac110ffbf9)
---
.github/workflows/ci-build-macos.yaml | 82 -------
.github/workflows/ci-cpp-build-centos7.yaml | 68 ------
...ci-cpp-build-windows.yaml => ci-cpp-build.yaml} | 135 +++++++++--
.github/workflows/ci-cpp-deb-pkg.yaml | 97 --------
.github/workflows/ci-cpp-rpm-pkg.yaml | 97 --------
.github/workflows/ci-go-functions-test.yaml | 85 -------
...o-functions-style.yaml => ci-go-functions.yaml} | 48 ++--
.github/workflows/ci-python-build-3.9-client.yaml | 68 ------
.github/workflows/pulsar-ci.yaml | 264 ++++++++++++++-------
9 files changed, 326 insertions(+), 618 deletions(-)
diff --git a/.github/workflows/ci-build-macos.yaml b/.github/workflows/ci-build-macos.yaml
deleted file mode 100644
index 0c73fddd005..00000000000
--- a/.github/workflows/ci-build-macos.yaml
+++ /dev/null
@@ -1,82 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Build - MacOS
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-
-jobs:
-
- build:
- name:
- runs-on: macos-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache Maven dependencies
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-all-
-
- - name: Set up JDK 17
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 17
-
-
- - name: build package
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B clean install -DskipTests
diff --git a/.github/workflows/ci-cpp-build-centos7.yaml b/.github/workflows/ci-cpp-build-centos7.yaml
deleted file mode 100644
index 5eb0ce02df8..00000000000
--- a/.github/workflows/ci-cpp-build-centos7.yaml
+++ /dev/null
@@ -1,68 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - CPP build on CentOS 7
-on:
- pull_request:
- branches:
- - master
- paths:
- - '.github/workflows/**'
- - 'pulsar-client-cpp/**'
- push:
- branches:
- - branch-*
- paths:
- - '.github/workflows/**'
- - 'pulsar-client-cpp/**'
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-jobs:
-
- cpp-build-centos7:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: build cpp client on centos 7
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- echo "Build C++ client library on CentOS 7"
- pulsar-client-cpp/docker-build-centos7.sh
diff --git a/.github/workflows/ci-cpp-build-windows.yaml b/.github/workflows/ci-cpp-build.yaml
similarity index 60%
rename from .github/workflows/ci-cpp-build-windows.yaml
rename to .github/workflows/ci-cpp-build.yaml
index a8a800e39d4..5e85b032d81 100644
--- a/.github/workflows/ci-cpp-build-windows.yaml
+++ b/.github/workflows/ci-cpp-build.yaml
@@ -17,7 +17,7 @@
# under the License.
#
-name: CI - CPP build on Windows
+name: CI - CPP build
on:
pull_request:
branches:
@@ -36,15 +36,55 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
-env:
- VCPKG_FEATURE_FLAGS: manifests
-
jobs:
- cpp-build-windows:
+ changed_files_job:
+ name: 'Changed files check'
+ runs-on: ubuntu-20.04
+ outputs:
+ docs_only: ${{ needs.changed_files_job.outputs.docs_only }}
+ cpp_only: ${{ needs.changed_files_job.outputs.cpp_only }}
+ changed_tests: ${{ steps.changes.outputs.tests_files }}
+ steps:
+ - name: checkout
+ uses: actions/checkout@v2
+
+ - name: Detect changed files
+ id: changes
+ uses: apache/pulsar-test-infra/paths-filter@master
+ with:
+ filters: .github/changes-filter.yaml
+ list-files: csv
+
+ - name: Check changed files
+ id: check_changes
+ run: |
+ echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
+
+ cpp-build-centos7:
+ needs: changed_files_job
+ name:
+ runs-on: ubuntu-latest
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
timeout-minutes: 120
+ steps:
+ - name: checkout
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ uses: ./.github/actions/tune-runner-vm
+
+ - name: build cpp client on centos 7
+ run: |
+ echo "Build C++ client library on CentOS 7"
+ pulsar-client-cpp/docker-build-centos7.sh
+
+ cpp-build-windows:
+ needs: changed_files_job
+ timeout-minutes: 120
name: ${{ matrix.name }}
runs-on: ${{ matrix.os }}
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
env:
VCPKG_ROOT: '${{ github.workspace }}/vcpkg'
strategy:
@@ -70,17 +110,6 @@ jobs:
- name: checkout
uses: actions/checkout@v2
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- name: Restore vcpkg and its artifacts.
uses: actions/cache@v2
id: vcpkg-cache
@@ -110,12 +139,11 @@ jobs:
shell: bash
- name: Install vcpkg packages
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
run: |
cd pulsar-client-cpp && ${{ env.VCPKG_ROOT }}\vcpkg.exe install --triplet ${{ matrix.triplet }}
- name: Configure (default)
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+
shell: bash
run: |
if [ "$RUNNER_OS" == "Windows" ]; then
@@ -130,7 +158,6 @@ jobs:
fi
- name: Compile
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
shell: bash
run: |
if [ "$RUNNER_OS" == "Windows" ]; then
@@ -139,7 +166,6 @@ jobs:
fi
- name: Configure (dynamic library only)
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
shell: bash
run: |
if [ "$RUNNER_OS" == "Windows" ]; then
@@ -155,10 +181,77 @@ jobs:
fi
- name: Compile
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
shell: bash
run: |
if [ "$RUNNER_OS" == "Windows" ]; then
cd pulsar-client-cpp && \
cmake --build ./build-1 --parallel --config Release
fi
+ cpp-deb-rpm-packaging:
+ needs: changed_files_job
+ name:
+ runs-on: ubuntu-latest
+ timeout-minutes: 120
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+
+ steps:
+ - name: checkout
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ uses: ./.github/actions/tune-runner-vm
+
+ - name: Cache local Maven repository
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/pulsar
+ key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-m2-dependencies-core-modules-
+
+ - name: Set up JDK 17
+ uses: actions/setup-java@v2
+ with:
+ distribution: 'temurin'
+ java-version: 17
+
+ - name: clean disk
+ run: |
+ sudo apt clean
+ docker rmi $(docker images -q) -f
+ df -h
+
+ - name: Package Pulsar source
+ run: mvn -B -ntp -q clean package -pl pulsar-client-api -am -DskipTests
+
+ - name: Build Debian packages
+ run: |
+ echo "Build Debian packages"
+ BUILD_IMAGE=1 pulsar-client-cpp/pkg/deb/docker-build-deb.sh
+
+ - name: Build RPM packages
+ run: |
+ echo "Build RPM packages"
+ BUILD_IMAGE=1 pulsar-client-cpp/pkg/rpm/docker-build-rpm.sh
+
+ build-python-wheel:
+ needs: changed_files_job
+ name:
+ runs-on: ubuntu-latest
+ timeout-minutes: 120
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+
+ steps:
+ - name: checkout
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ uses: ./.github/actions/tune-runner-vm
+
+ - name: build python3.9 client
+
+ run: |
+ echo "Build Python3.9 client library"
+ pulsar-client-cpp/docker-build-python3.9.sh
\ No newline at end of file
diff --git a/.github/workflows/ci-cpp-deb-pkg.yaml b/.github/workflows/ci-cpp-deb-pkg.yaml
deleted file mode 100644
index b6c75c4c727..00000000000
--- a/.github/workflows/ci-cpp-deb-pkg.yaml
+++ /dev/null
@@ -1,97 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - CPP DEB packaging
-on:
- pull_request:
- branches:
- - master
- paths:
- - '.github/workflows/**'
- - 'pulsar-client-cpp/**'
- push:
- branches:
- - branch-*
- paths:
- - '.github/workflows/**'
- - 'pulsar-client-cpp/**'
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-jobs:
-
- cpp-deb-packaging:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 17
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 17
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: Package Pulsar source
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -ntp -q clean package -pl pulsar-client-api -am -DskipTests
-
- - name: Build Debian packages
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- echo "Build Debian packages"
- BUILD_IMAGE=1 pulsar-client-cpp/pkg/deb/docker-build-deb.sh
diff --git a/.github/workflows/ci-cpp-rpm-pkg.yaml b/.github/workflows/ci-cpp-rpm-pkg.yaml
deleted file mode 100644
index b8da1eac99d..00000000000
--- a/.github/workflows/ci-cpp-rpm-pkg.yaml
+++ /dev/null
@@ -1,97 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - CPP RPM packaging
-on:
- pull_request:
- branches:
- - master
- paths:
- - '.github/workflows/**'
- - 'pulsar-client-cpp/**'
- push:
- branches:
- - branch-*
- paths:
- - '.github/workflows/**'
- - 'pulsar-client-cpp/**'
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-jobs:
-
- cpp-rpm-packaging:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 17
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 17
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: Package Pulsar source
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -ntp -q clean package -pl pulsar-client-api -am -DskipTests
-
- - name: Build RPM packages
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- echo "Build RPM packages"
- BUILD_IMAGE=1 pulsar-client-cpp/pkg/rpm/docker-build-rpm.sh
diff --git a/.github/workflows/ci-go-functions-test.yaml b/.github/workflows/ci-go-functions-test.yaml
deleted file mode 100644
index 0efc4d2ca2c..00000000000
--- a/.github/workflows/ci-go-functions-test.yaml
+++ /dev/null
@@ -1,85 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Go Functions Tests
-on:
- pull_request:
- branches:
- - master
- paths:
- - '.github/workflows/**'
- - 'pulsar-function-go/**'
- push:
- branches:
- - branch-*
- paths:
- - '.github/workflows/**'
- - 'pulsar-function-go/**'
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-
-jobs:
-
- go-functions-tests:
- name: Run Go Tests
- strategy:
- matrix:
- go-version: [1.15, 1.16, 1.17]
- platform: [ubuntu-latest]
- runs-on: ${{ matrix.platform }}
- timeout-minutes: 120
-
- steps:
- - name: Check out code into the Go module directory
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: |
- echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
-
- - name: Set up Go
- uses: actions/setup-go@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' && steps.check_changes.outputs.cpp_only != 'true' }}
- with:
- go-version: ${{ matrix.go-version }}
- id: go
-
- - name: Run tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' && steps.check_changes.outputs.cpp_only != 'true' }}
- run: |
- cd pulsar-function-go
- export GO111MODULE=on
- go test -v $(go list ./... | grep -v examples)
diff --git a/.github/workflows/ci-go-functions-style.yaml b/.github/workflows/ci-go-functions.yaml
similarity index 78%
rename from .github/workflows/ci-go-functions-style.yaml
rename to .github/workflows/ci-go-functions.yaml
index d9ee7b15637..8fa6b0a2b3b 100644
--- a/.github/workflows/ci-go-functions-style.yaml
+++ b/.github/workflows/ci-go-functions.yaml
@@ -17,7 +17,7 @@
# under the License.
#
-name: CI - Go Functions style check
+name: CI - Go Functions
on:
pull_request:
branches:
@@ -40,23 +40,19 @@ env:
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
jobs:
- check-style:
-
- name: Go ${{ matrix.go-version }} Functions style check
- runs-on: ubuntu-latest
- strategy:
- matrix:
- go-version: [1.15, 1.16, 1.17]
-
+ changed_files_job:
+ name: 'Changed files check'
+ runs-on: ubuntu-20.04
+ outputs:
+ docs_only: ${{ steps.check_changes.outputs.docs_only }}
+ cpp_only: ${{ steps.check_changes.outputs.cpp_only }}
+ changed_tests: ${{ steps.changes.outputs.tests_files }}
steps:
- - name: Check out code into the Go module directory
+ - name: checkout
uses: actions/checkout@v2
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- name: Detect changed files
- id: changes
+ id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
@@ -66,32 +62,44 @@ jobs:
id: check_changes
run: |
echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
- echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
+
+ check-style:
+ needs: changed_files_job
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ name: Go ${{ matrix.go-version }} Functions style check
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ go-version: [1.15, 1.16, 1.17]
+
+ steps:
+ - name: Check out code into the Go module directory
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ uses: ./.github/actions/tune-runner-vm
- name: Set up Go
uses: actions/setup-go@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' && steps.check_changes.outputs.cpp_only != 'true' }}
with:
go-version: ${{ matrix.go-version }}
id: go
- name: InstallTool
- if: ${{ steps.check_changes.outputs.docs_only != 'true' && steps.check_changes.outputs.cpp_only != 'true' }}
run: |
cd pulsar-function-go
wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.18.0
./bin/golangci-lint --version
- name: Build
- if: ${{ steps.check_changes.outputs.docs_only != 'true' && steps.check_changes.outputs.cpp_only != 'true' }}
run: |
cd pulsar-function-go
export GO111MODULE=on
go build ./...
- - name: CheckStyle
- if: ${{ steps.check_changes.outputs.docs_only != 'true' && steps.check_changes.outputs.cpp_only != 'true' }}
+ - name: Test
run: |
cd pulsar-function-go
export GO111MODULE=on
./bin/golangci-lint run -c ./golangci.yml ./pf
+ go test -v $(go list ./... | grep -v examples)
diff --git a/.github/workflows/ci-python-build-3.9-client.yaml b/.github/workflows/ci-python-build-3.9-client.yaml
deleted file mode 100644
index a1ef4a73d7b..00000000000
--- a/.github/workflows/ci-python-build-3.9-client.yaml
+++ /dev/null
@@ -1,68 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Python - Build 3.9 client
-on:
- pull_request:
- branches:
- - master
- paths:
- - '.github/workflows/**'
- - 'pulsar-client-cpp/**'
- push:
- branches:
- - branch-*
- paths:
- - '.github/workflows/**'
- - 'pulsar-client-cpp/**'
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-jobs:
-
- build-wheel:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: build python3.9 client
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- echo "Build Python3.9 client library"
- pulsar-client-cpp/docker-build-python3.9.sh
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index 65698a014c3..13cbe7e1a76 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -67,25 +67,22 @@ jobs:
name: Build and License check
runs-on: ubuntu-20.04
timeout-minutes: 60
-
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
steps:
- name: checkout
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: actions/checkout@v2
- name: Tune Runner VM
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
- if: ${{ github.repository != 'apache/pulsar' && needs.changed_files_job.outputs.docs_only != 'true' }}
+ if: ${{ github.repository != 'apache/pulsar' }}
uses: ./.github/actions/ssh-access
with:
limit-access-to-actor: true
- name: Cache local Maven repository
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: actions/cache@v2
with:
path: |
@@ -97,30 +94,24 @@ jobs:
- name: Set up JDK 17
uses: actions/setup-java@v2
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
with:
distribution: 'temurin'
java-version: 17
- name: Check source code license headers
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
run: mvn -B -T 8 -ntp initialize apache-rat:check license:check
- name: Build core-modules
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
run: |
mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true
- name: Check binary licenses
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz
- name: Install gh-actions-artifact-client.js
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Save maven build results to Github artifact cache so that the results can be reused
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \
@@ -135,11 +126,45 @@ jobs:
with:
action: wait
+ macos-build:
+ name:
+ runs-on: macos-latest
+ timeout-minutes: 120
+ needs: [ 'changed_files_job', 'build-and-license-check' ]
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ steps:
+ - name: checkout
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ uses: ./.github/actions/tune-runner-vm
+
+ - name: Cache Maven dependencies
+ if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/pulsar
+ key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-m2-dependencies-all-
+
+ - name: Set up JDK 17
+ uses: actions/setup-java@v2
+ with:
+ distribution: 'temurin'
+ java-version: 17
+
+ - name: build package
+ run: mvn -B clean package -DskipTests -T 1C -ntp
+
unit-tests:
name: CI - Unit - ${{ matrix.name }}
runs-on: ubuntu-20.04
timeout-minutes: ${{ matrix.timeout || 60 }}
needs: ['changed_files_job', 'build-and-license-check']
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
strategy:
fail-fast: false
matrix:
@@ -164,22 +189,19 @@ jobs:
steps:
- name: checkout
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
uses: actions/checkout@v2
- name: Tune Runner VM
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
- if: ${{ github.repository != 'apache/pulsar' && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ github.repository != 'apache/pulsar' }}
uses: ./.github/actions/ssh-access
with:
limit-access-to-actor: true
- name: Cache Maven dependencies
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
uses: actions/cache@v2
with:
path: |
@@ -191,28 +213,23 @@ jobs:
- name: Set up JDK ${{ matrix.jdk || '17' }}
uses: actions/setup-java@v2
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
with:
distribution: 'temurin'
java-version: ${{ matrix.jdk || '17' }}
- name: Install gh-actions-artifact-client.js
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Restore maven build results from Github artifact cache
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
- name: Run setup commands
- if: ${{ matrix.setup && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
run: |
${{ matrix.setup }}
- name: Run unit test group '${{ matrix.group }}'
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
run: |
CHANGED_TESTS="${{ needs.changed_files_job.outputs.tests_files }}" ./build/run_unit_group.sh ${{ matrix.group }}
@@ -221,19 +238,19 @@ jobs:
run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps
- name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
- if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ always() }}
uses: ./.github/actions/copy-test-reports
- name: Publish Test Report
uses: apache/pulsar-test-infra/action-junit-report@master
- if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ always() }}
with:
report_paths: 'test-reports/TEST-*.xml'
annotate_only: 'true'
- name: Upload Surefire reports
uses: actions/upload-artifact@v3
- if: ${{ !success() && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ !success() }}
with:
name: Unit-${{ matrix.group }}-surefire-reports
path: surefire-reports
@@ -241,7 +258,7 @@ jobs:
- name: Upload possible heap dump
uses: actions/upload-artifact@v3
- if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ always() }}
with:
name: Unit-${{ matrix.group }}-heapdump
path: /tmp/*.hprof
@@ -261,26 +278,23 @@ jobs:
runs-on: ubuntu-20.04
timeout-minutes: 60
needs: ['changed_files_job', 'build-and-license-check']
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
env:
UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
steps:
- name: checkout
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
uses: actions/checkout@v2
- name: Tune Runner VM
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
- if: ${{ github.repository != 'apache/pulsar' && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
uses: ./.github/actions/ssh-access
with:
limit-access-to-actor: true
- name: Cache Maven dependencies
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
uses: actions/cache@v2
with:
path: |
@@ -292,23 +306,19 @@ jobs:
- name: Set up JDK 17
uses: actions/setup-java@v2
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
with:
distribution: 'temurin'
java-version: 17
- name: Install gh-actions-artifact-client.js
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: restore maven build results from Github artifact cache
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
- name: Build java-test-image docker image
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
run: |
# build docker image
mvn -B -am -pl tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
@@ -316,7 +326,6 @@ jobs:
-Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
- name: save docker image apachepulsar/java-test-image:latest to Github artifact cache
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image
@@ -333,6 +342,7 @@ jobs:
runs-on: ubuntu-20.04
timeout-minutes: ${{ matrix.timeout || 60 }}
needs: ['changed_files_job', 'pulsar-java-test-image']
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
env:
PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
strategy:
@@ -370,22 +380,19 @@ jobs:
steps:
- name: checkout
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
uses: actions/checkout@v2
- name: Tune Runner VM
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
- if: ${{ github.repository != 'apache/pulsar' && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ github.repository != 'apache/pulsar' }}
uses: ./.github/actions/ssh-access
with:
limit-access-to-actor: true
- name: Cache Maven dependencies
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
uses: actions/cache@v2
with:
path: |
@@ -397,40 +404,35 @@ jobs:
- name: Set up JDK 17
uses: actions/setup-java@v2
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
with:
distribution: 'temurin'
java-version: 17
- name: Install gh-actions-artifact-client.js
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Restore maven build results from Github artifact cache
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
- name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image
- name: Run setup commands
- if: ${{ matrix.setup && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ matrix.setup }}
run: |
${{ matrix.setup }}
- name: Set up runtime JDK ${{ matrix.runtime_jdk }}
uses: actions/setup-java@v2
- if: ${{ matrix.runtime_jdk && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ matrix.runtime_jdk }}
with:
distribution: 'temurin'
java-version: ${{ matrix.runtime_jdk }}
- name: Run integration test group '${{ matrix.group }}'
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
run: |
./build/run_integration_group.sh ${{ matrix.group }}
@@ -439,19 +441,19 @@ jobs:
run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps
- name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
- if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ always() }}
uses: ./.github/actions/copy-test-reports
- name: Publish Test Report
uses: apache/pulsar-test-infra/action-junit-report@master
- if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ always() }}
with:
report_paths: 'test-reports/TEST-*.xml'
annotate_only: 'true'
- name: Upload Surefire reports
uses: actions/upload-artifact@v3
- if: ${{ !success() && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ !success() }}
with:
name: Integration-${{ matrix.group }}-surefire-reports
path: surefire-reports
@@ -459,7 +461,7 @@ jobs:
- name: Upload container logs
uses: actions/upload-artifact@v3
- if: ${{ !success() && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ !success() }}
continue-on-error: true
with:
name: Integration-${{ matrix.group }}-container-logs
@@ -502,30 +504,27 @@ jobs:
runs-on: ubuntu-20.04
timeout-minutes: 60
needs: ['changed_files_job', 'build-and-license-check']
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
env:
UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
steps:
- name: checkout
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: actions/checkout@v2
- name: Tune Runner VM
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
- if: ${{ github.repository != 'apache/pulsar' && needs.changed_files_job.outputs.docs_only != 'true' }}
+ if: ${{ github.repository != 'apache/pulsar' }}
uses: ./.github/actions/ssh-access
with:
limit-access-to-actor: true
- name: Clean Disk
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: ./.github/actions/clean-disk
- name: Cache local Maven repository
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: actions/cache@v2
with:
path: |
@@ -538,23 +537,19 @@ jobs:
- name: Set up JDK 17
uses: actions/setup-java@v2
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
with:
distribution: 'temurin'
java-version: 17
- name: Install gh-actions-artifact-client.js
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: restore maven build results from Github artifact cache
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
- name: Build latest-version-image docker image
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
run: |
# build docker image
# include building of Pulsar SQL, Connectors, Offloaders and server distros
@@ -563,7 +558,6 @@ jobs:
-Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
- name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image
@@ -580,6 +574,7 @@ jobs:
runs-on: ubuntu-20.04
timeout-minutes: 60
needs: ['changed_files_job', 'pulsar-test-latest-version-image']
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
env:
PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
strategy:
@@ -607,30 +602,135 @@ jobs:
- name: Pulsar IO
group: PULSAR_IO
+ - name: Sql
+ group: SQL
+
+ steps:
+ - name: checkout
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ uses: ./.github/actions/tune-runner-vm
+
+ - name: Setup ssh access to build runner VM
+ # ssh access is enabled for builds in own forks
+ if: ${{ github.repository != 'apache/pulsar' }}
+ uses: ./.github/actions/ssh-access
+ with:
+ limit-access-to-actor: true
+
+ - name: Cache local Maven repository
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/pulsar
+ key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
+ ${{ runner.os }}-m2-dependencies-core-modules-
+
+ - name: Set up JDK 17
+ uses: actions/setup-java@v2
+ with:
+ distribution: 'temurin'
+ java-version: 17
+
+ - name: Install gh-actions-artifact-client.js
+ uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
+
+ - name: Restore maven build results from Github artifact cache
+ run: |
+ cd $HOME
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
+
+ - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
+ run: |
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image
+
+ - name: Run setup commands
+ if: ${{ matrix.setup }}
+ run: |
+ ${{ matrix.setup }}
+
+ - name: Run system test group '${{ matrix.group }}'
+ run: |
+ ./build/run_integration_group.sh ${{ matrix.group }}
+
+ - name: print JVM thread dumps when cancelled
+ if: cancelled()
+ run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps
+
+ - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
+ if: ${{ always() }}
+ uses: ./.github/actions/copy-test-reports
+
+ - name: Publish Test Report
+ uses: apache/pulsar-test-infra/action-junit-report@master
+ if: ${{ always() }}
+ with:
+ report_paths: 'test-reports/TEST-*.xml'
+ annotate_only: 'true'
+
+ - name: Upload container logs
+ uses: actions/upload-artifact@v3
+ if: ${{ !success() }}
+ continue-on-error: true
+ with:
+ name: System-${{ matrix.group }}-container-logs
+ path: tests/integration/target/container-logs
+ retention-days: 7
+
+ - name: Upload Surefire reports
+ uses: actions/upload-artifact@v3
+ if: ${{ !success() }}
+ with:
+ name: System-${{ matrix.name }}-surefire-reports
+ path: surefire-reports
+ retention-days: 7
+
+ - name: Wait for ssh connection when build fails
+ # ssh access is enabled for builds in own forks
+ uses: ./.github/actions/ssh-access
+ if: ${{ failure() && github.repository != 'apache/pulsar' }}
+ continue-on-error: true
+ with:
+ action: wait
+
+ flaky-system-tests:
+ name: CI Flaky - System - ${{ matrix.name }}
+ runs-on: ubuntu-20.04
+ timeout-minutes: 60
+ needs: [ 'changed_files_job', 'pulsar-test-latest-version-image' ]
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ env:
+ PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - name: Plugin
+ group: PLUGIN
+
- name: Pulsar IO - Oracle
group: PULSAR_IO_ORA
- - name: Sql
- group: SQL
steps:
- name: checkout
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: actions/checkout@v2
- name: Tune Runner VM
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
- if: ${{ github.repository != 'apache/pulsar' && needs.changed_files_job.outputs.docs_only != 'true' }}
+ if: ${{ github.repository != 'apache/pulsar' }}
uses: ./.github/actions/ssh-access
with:
limit-access-to-actor: true
- name: Cache local Maven repository
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: actions/cache@v2
with:
path: |
@@ -643,33 +743,28 @@ jobs:
- name: Set up JDK 17
uses: actions/setup-java@v2
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
with:
distribution: 'temurin'
java-version: 17
- name: Install gh-actions-artifact-client.js
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Restore maven build results from Github artifact cache
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
- name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image
- name: Run setup commands
- if: ${{ matrix.setup && needs.changed_files_job.outputs.docs_only != 'true' }}
+ if: ${{ matrix.setup }}
run: |
${{ matrix.setup }}
- name: Run system test group '${{ matrix.group }}'
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
run: |
./build/run_integration_group.sh ${{ matrix.group }}
@@ -678,19 +773,19 @@ jobs:
run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps
- name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
- if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ always() }}
uses: ./.github/actions/copy-test-reports
- name: Publish Test Report
uses: apache/pulsar-test-infra/action-junit-report@master
- if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ always() }}
with:
report_paths: 'test-reports/TEST-*.xml'
annotate_only: 'true'
- name: Upload container logs
uses: actions/upload-artifact@v3
- if: ${{ !success() && needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }}
+ if: ${{ !success() }}
continue-on-error: true
with:
name: System-${{ matrix.group }}-container-logs
@@ -699,7 +794,7 @@ jobs:
- name: Upload Surefire reports
uses: actions/upload-artifact@v3
- if: ${{ !success() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ if: ${{ !success() }}
with:
name: System-${{ matrix.name }}-surefire-reports
path: surefire-reports
@@ -719,7 +814,8 @@ jobs:
timeout-minutes: 10
needs: [
'changed_files_job',
- 'system-tests'
+ 'system-tests',
+ 'flaky-system-tests'
]
if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
steps:
@@ -736,27 +832,35 @@ jobs:
run: |
gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst
- delete-pulsar-build-binaries-artifact:
- name: "Delete pulsar-maven-repository-binaries.tar.zst artifact"
+ # This job is required for pulls to be merged.
+ # It depends on all other jobs in this workflow.
+ # It cleans up the binaries in the same job in order to not spin up another runner for basically doing nothing.
+ pulsar-ci-checks-completed:
+ name: "Pulsar CI checks completed"
runs-on: ubuntu-20.04
timeout-minutes: 10
needs: [
'changed_files_job',
'unit-tests',
'integration-tests',
- 'system-tests'
+ 'system-tests',
+ 'macos-build'
]
- if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ if: always()
steps:
- name: checkout
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: actions/checkout@v2
- name: Tune Runner VM
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: ./.github/actions/tune-runner-vm
- name: Install gh-actions-artifact-client.js
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Delete maven repository binaries from GitHub Actions Artifacts
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
run: |
- gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst
+ gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst
\ No newline at end of file