You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by la...@apache.org on 2023/12/22 08:43:08 UTC
(incubator-pegasus) 01/01: chore(CI): Improve github actions (2nd)
This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch ci_refactor_more
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
commit 5a37c6a49d30e4cbf1819bd56f2e284265f4c336
Author: Yingchun Lai <la...@apache.org>
AuthorDate: Fri Dec 22 16:42:45 2023 +0800
chore(CI): Improve github actions (2nd)
---
.github/actions/download_artifact/action.yaml | 5 +-
.../rebuild_thirdparty_if_needed/action.yaml | 9 +--
.../action.yaml | 18 ++----
.github/workflows/lint_and_test_cpp.yaml | 66 ++++++++++++++--------
.github/workflows/lint_and_test_java-client.yml | 12 ++--
.github/workflows/regular-build.yml | 27 ++-------
6 files changed, 64 insertions(+), 73 deletions(-)
diff --git a/.github/actions/download_artifact/action.yaml b/.github/actions/download_artifact/action.yaml
index 35ddb8ae0..29d68973c 100644
--- a/.github/actions/download_artifact/action.yaml
+++ b/.github/actions/download_artifact/action.yaml
@@ -20,10 +20,7 @@ runs:
using: composite
steps:
- name: Unpack prebuilt third-parties
- run: |
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- shell: bash
+ uses: ".github/actions/unpack_prebuilt_thirdparties/action.yaml"
- name: Download tarball
uses: actions/download-artifact@v3
with:
diff --git a/.github/actions/rebuild_thirdparty_if_needed/action.yaml b/.github/actions/rebuild_thirdparty_if_needed/action.yaml
index ff48fff29..035ae47d5 100644
--- a/.github/actions/rebuild_thirdparty_if_needed/action.yaml
+++ b/.github/actions/rebuild_thirdparty_if_needed/action.yaml
@@ -30,14 +30,7 @@ runs:
- 'thirdparty/**'
- name: Unpack prebuilt third-parties
if: steps.changes.outputs.thirdparty == 'false'
- run: |
- rm -f /root/thirdparties-src.zip
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- rm -rf ./thirdparty/hadoop-bin/share/doc
- rm -rf ./thirdparty/zookeeper-bin/docs
- shell: bash
+ uses: ".github/actions/unpack_prebuilt_thirdparties/action.yaml"
- name: Rebuild third-parties
if: steps.changes.outputs.thirdparty == 'true'
working-directory: thirdparty
diff --git a/.github/actions/download_artifact/action.yaml b/.github/actions/unpack_prebuilt_thirdparties/action.yaml
similarity index 71%
copy from .github/actions/download_artifact/action.yaml
copy to .github/actions/unpack_prebuilt_thirdparties/action.yaml
index 35ddb8ae0..0e1b2cdaf 100644
--- a/.github/actions/download_artifact/action.yaml
+++ b/.github/actions/unpack_prebuilt_thirdparties/action.yaml
@@ -15,22 +15,16 @@
# specific language governing permissions and limitations
# under the License.
-name: Download artifact
+name: Unpack prebuilt third-parties
runs:
using: composite
steps:
- - name: Unpack prebuilt third-parties
+ - name: Unpack
run: |
+ rm -f /root/thirdparties-src.zip
unzip /root/thirdparties-bin.zip -d ./thirdparty
rm -f /root/thirdparties-bin.zip
- shell: bash
- - name: Download tarball
- uses: actions/download-artifact@v3
- with:
- name: ${{ env.ARTIFACT_NAME }}_artifact_${{ github.sha }}
- path: .
- - name: Unpack tarball
- run: |
- tar -zxvf ${ARTIFACT_NAME}_builder.tar
- rm -f ${ARTIFACT_NAME}_builder.tar
+ find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
+ rm -rf ./thirdparty/hadoop-bin/share/doc
+ rm -rf ./thirdparty/zookeeper-bin/docs
shell: bash
diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml
index f90a18074..48abc8e19 100644
--- a/.github/workflows/lint_and_test_cpp.yaml
+++ b/.github/workflows/lint_and_test_cpp.yaml
@@ -72,7 +72,8 @@ jobs:
- name: Free Disk Space (Ubuntu)
run: |
.github/workflows/free_disk_space.sh
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
- name: Build IWYU binary
run: |
mkdir iwyu && cd iwyu && git clone https://github.com/include-what-you-use/include-what-you-use.git
@@ -97,10 +98,14 @@ jobs:
container:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
steps:
- - uses: actions/checkout@v3
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
- - uses: "./.github/actions/build_pegasus"
- - uses: "./.github/actions/upload_artifact"
+ - name: Clone code
+ uses: actions/checkout@v3
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
+ - name: Upload artifact
+ uses: "./.github/actions/upload_artifact"
test_Release:
name: Test Release
@@ -157,8 +162,10 @@ jobs:
options: --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/download_artifact"
- - uses: "./.github/actions/run_server_tests"
+ - name: Download artifact
+ uses: "./.github/actions/download_artifact"
+ - name: Run server tests
+ uses: "./.github/actions/run_server_tests"
build_ASAN:
name: Build ASAN
@@ -172,9 +179,12 @@ jobs:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
- - uses: "./.github/actions/build_pegasus"
- - uses: "./.github/actions/upload_artifact"
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
+ - name: Upload artifact
+ uses: "./.github/actions/upload_artifact"
test_ASAN:
name: Test ASAN
@@ -233,8 +243,10 @@ jobs:
options: --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/download_artifact"
- - uses: "./.github/actions/run_server_tests"
+ - name: Download artifact
+ uses: "./.github/actions/download_artifact"
+ - name: Run server tests
+ uses: "./.github/actions/run_server_tests"
# TODO(yingchun): Build and test UBSAN version would cost a very long time, we will run these tests
# when we are going to release a stable version. So we disable them in regular CI
@@ -251,9 +263,12 @@ jobs:
# image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
# steps:
# - uses: actions/checkout@v3
-# - uses: "./.github/actions/rebuild_thirdparty_if_needed"
-# - uses: "./.github/actions/build_pegasus"
-# - uses: "./.github/actions/upload_artifact"
+# - name: Rebuild thirdparty if needed
+# uses: "./.github/actions/rebuild_thirdparty_if_needed"
+# - name: Build Pegasus
+# uses: "./.github/actions/build_pegasus"
+# - name: Upload artifact
+# uses: "./.github/actions/upload_artifact"
#
# test_UBSAN:
# name: Test UBSAN
@@ -308,8 +323,10 @@ jobs:
# options: --cap-add=SYS_PTRACE
# steps:
# - uses: actions/checkout@v3
-# - uses: "./.github/actions/download_artifact"
-# - uses: "./.github/actions/run_server_tests"
+# - name: Download artifact
+# uses: "./.github/actions/download_artifact"
+# - name: Run server tests
+# uses: "./.github/actions/run_server_tests"
build_with_jemalloc:
name: Build with jemalloc
@@ -323,10 +340,13 @@ jobs:
image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }}
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
# TODO(yingchun): Append "-m dsn_utils_tests" to the command if not needed to pack server or tools, for example, the dependencies are static linked.
- - uses: "./.github/actions/build_pegasus"
- - uses: "./.github/actions/upload_artifact"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
+ - name: Upload artifact
+ uses: "./.github/actions/upload_artifact"
test_with_jemalloc:
name: Test with jemallc
@@ -344,8 +364,10 @@ jobs:
options: --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/download_artifact"
- - uses: "./.github/actions/run_server_tests"
+ - name: Download artifact
+ uses: "./.github/actions/download_artifact"
+ - name: Run server tests
+ uses: "./.github/actions/run_server_tests"
build_pegasus_on_macos:
name: macOS
diff --git a/.github/workflows/lint_and_test_java-client.yml b/.github/workflows/lint_and_test_java-client.yml
index dfe747bd8..119877d0e 100644
--- a/.github/workflows/lint_and_test_java-client.yml
+++ b/.github/workflows/lint_and_test_java-client.yml
@@ -59,9 +59,12 @@ jobs:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
- - uses: "./.github/actions/build_pegasus"
- - uses: "./.github/actions/upload_artifact"
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
+ - name: Upload artifact
+ uses: "./.github/actions/upload_artifact"
test_java_client:
name: Test Java client
@@ -86,7 +89,8 @@ jobs:
- uses: actions/setup-java@v1
with:
java-version: ${{ matrix.java }}
- - uses: "./.github/actions/download_artifact"
+ - name: Download artifact
+ uses: "./.github/actions/download_artifact"
- name: Start Pegasus cluster
run: |
apt-get update
diff --git a/.github/workflows/regular-build.yml b/.github/workflows/regular-build.yml
index f536479be..da20c54d8 100644
--- a/.github/workflows/regular-build.yml
+++ b/.github/workflows/regular-build.yml
@@ -72,30 +72,11 @@ jobs:
working-directory: /root/incubator-pegasus
steps:
- name: Clone Apache Pegasus Source
- working-directory: /root
- run: |
- git clone -b ${{ github.ref_name }} --depth=1 https://github.com/apache/incubator-pegasus.git
- - name: Free Disk Space (Ubuntu)
- run: |
- .github/workflows/free_disk_space.sh
+ uses: actions/checkout@v3
- name: Unpack prebuilt third-parties
- run: |
- rm -f /root/thirdparties-src.zip
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- rm -rf ./thirdparty/hadoop-bin/share/doc
- rm -rf ./thirdparty/zookeeper-bin/docs
- - name: Compilation Pegasus
- run: |
- ./run.sh build --test --compiler ${{ matrix.compiler }} --skip_thirdparty -j $(nproc)
- - name: Clear Build Files
- run: |
- find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- - name: Packaging Server
- run: |
- ./run.sh pack_server
- rm -rf pegasus-server-*
+ uses: ".github/actions/unpack_prebuilt_thirdparties/action.yaml"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
build_and_lint_go:
name: Build and Lint Golang
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org