You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by la...@apache.org on 2024/01/02 03:02:04 UTC
(incubator-pegasus) branch master updated: chore(CI): Improve github actions (2nd) (#1812)
This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 394617b77 chore(CI): Improve github actions (2nd) (#1812)
394617b77 is described below
commit 394617b778538c1842f2423f48dd763bfda09d85
Author: Yingchun Lai <la...@apache.org>
AuthorDate: Tue Jan 2 11:01:59 2024 +0800
chore(CI): Improve github actions (2nd) (#1812)
- Make the "Unpack prebuilt third-parties" as a independent job
- Add names for the jobs which calling the action yamls
---
.github/actions/download_artifact/action.yaml | 5 +-
.../rebuild_thirdparty_if_needed/action.yaml | 9 +--
.../action.yaml | 18 ++----
.github/workflows/lint_and_test_cpp.yaml | 66 ++++++++++++++--------
.github/workflows/lint_and_test_java-client.yml | 12 ++--
.github/workflows/regular-build.yml | 27 ++-------
6 files changed, 64 insertions(+), 73 deletions(-)
diff --git a/.github/actions/download_artifact/action.yaml b/.github/actions/download_artifact/action.yaml
index 35ddb8ae0..07cf2d476 100644
--- a/.github/actions/download_artifact/action.yaml
+++ b/.github/actions/download_artifact/action.yaml
@@ -20,10 +20,7 @@ runs:
using: composite
steps:
- name: Unpack prebuilt third-parties
- run: |
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- shell: bash
+ uses: "./.github/actions/unpack_prebuilt_thirdparties"
- name: Download tarball
uses: actions/download-artifact@v3
with:
diff --git a/.github/actions/rebuild_thirdparty_if_needed/action.yaml b/.github/actions/rebuild_thirdparty_if_needed/action.yaml
index ff48fff29..6e64f7db4 100644
--- a/.github/actions/rebuild_thirdparty_if_needed/action.yaml
+++ b/.github/actions/rebuild_thirdparty_if_needed/action.yaml
@@ -30,14 +30,7 @@ runs:
- 'thirdparty/**'
- name: Unpack prebuilt third-parties
if: steps.changes.outputs.thirdparty == 'false'
- run: |
- rm -f /root/thirdparties-src.zip
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- rm -rf ./thirdparty/hadoop-bin/share/doc
- rm -rf ./thirdparty/zookeeper-bin/docs
- shell: bash
+ uses: "./.github/actions/unpack_prebuilt_thirdparties"
- name: Rebuild third-parties
if: steps.changes.outputs.thirdparty == 'true'
working-directory: thirdparty
diff --git a/.github/actions/download_artifact/action.yaml b/.github/actions/unpack_prebuilt_thirdparties/action.yaml
similarity index 71%
copy from .github/actions/download_artifact/action.yaml
copy to .github/actions/unpack_prebuilt_thirdparties/action.yaml
index 35ddb8ae0..0e1b2cdaf 100644
--- a/.github/actions/download_artifact/action.yaml
+++ b/.github/actions/unpack_prebuilt_thirdparties/action.yaml
@@ -15,22 +15,16 @@
# specific language governing permissions and limitations
# under the License.
-name: Download artifact
+name: Unpack prebuilt third-parties
runs:
using: composite
steps:
- - name: Unpack prebuilt third-parties
+ - name: Unpack
run: |
+ rm -f /root/thirdparties-src.zip
unzip /root/thirdparties-bin.zip -d ./thirdparty
rm -f /root/thirdparties-bin.zip
- shell: bash
- - name: Download tarball
- uses: actions/download-artifact@v3
- with:
- name: ${{ env.ARTIFACT_NAME }}_artifact_${{ github.sha }}
- path: .
- - name: Unpack tarball
- run: |
- tar -zxvf ${ARTIFACT_NAME}_builder.tar
- rm -f ${ARTIFACT_NAME}_builder.tar
+ find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
+ rm -rf ./thirdparty/hadoop-bin/share/doc
+ rm -rf ./thirdparty/zookeeper-bin/docs
shell: bash
diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml
index 324bf00c6..a17121972 100644
--- a/.github/workflows/lint_and_test_cpp.yaml
+++ b/.github/workflows/lint_and_test_cpp.yaml
@@ -72,7 +72,8 @@ jobs:
- name: Free Disk Space (Ubuntu)
run: |
.github/workflows/free_disk_space.sh
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
- name: Build IWYU binary
run: |
mkdir iwyu && cd iwyu && git clone https://github.com/include-what-you-use/include-what-you-use.git
@@ -97,10 +98,14 @@ jobs:
container:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
steps:
- - uses: actions/checkout@v3
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
- - uses: "./.github/actions/build_pegasus"
- - uses: "./.github/actions/upload_artifact"
+ - name: Clone code
+ uses: actions/checkout@v3
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
+ - name: Upload artifact
+ uses: "./.github/actions/upload_artifact"
test_Release:
name: Test Release
@@ -158,8 +163,10 @@ jobs:
options: --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/download_artifact"
- - uses: "./.github/actions/run_server_tests"
+ - name: Download artifact
+ uses: "./.github/actions/download_artifact"
+ - name: Run server tests
+ uses: "./.github/actions/run_server_tests"
build_ASAN:
name: Build ASAN
@@ -173,9 +180,12 @@ jobs:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
- - uses: "./.github/actions/build_pegasus"
- - uses: "./.github/actions/upload_artifact"
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
+ - name: Upload artifact
+ uses: "./.github/actions/upload_artifact"
test_ASAN:
name: Test ASAN
@@ -235,8 +245,10 @@ jobs:
options: --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/download_artifact"
- - uses: "./.github/actions/run_server_tests"
+ - name: Download artifact
+ uses: "./.github/actions/download_artifact"
+ - name: Run server tests
+ uses: "./.github/actions/run_server_tests"
# TODO(yingchun): Build and test UBSAN version would cost a very long time, we will run these tests
# when we are going to release a stable version. So we disable them in regular CI
@@ -253,9 +265,12 @@ jobs:
# image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
# steps:
# - uses: actions/checkout@v3
-# - uses: "./.github/actions/rebuild_thirdparty_if_needed"
-# - uses: "./.github/actions/build_pegasus"
-# - uses: "./.github/actions/upload_artifact"
+# - name: Rebuild thirdparty if needed
+# uses: "./.github/actions/rebuild_thirdparty_if_needed"
+# - name: Build Pegasus
+# uses: "./.github/actions/build_pegasus"
+# - name: Upload artifact
+# uses: "./.github/actions/upload_artifact"
#
# test_UBSAN:
# name: Test UBSAN
@@ -311,8 +326,10 @@ jobs:
# options: --cap-add=SYS_PTRACE
# steps:
# - uses: actions/checkout@v3
-# - uses: "./.github/actions/download_artifact"
-# - uses: "./.github/actions/run_server_tests"
+# - name: Download artifact
+# uses: "./.github/actions/download_artifact"
+# - name: Run server tests
+# uses: "./.github/actions/run_server_tests"
build_with_jemalloc:
name: Build with jemalloc
@@ -326,10 +343,13 @@ jobs:
image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }}
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
# TODO(yingchun): Append "-m dsn_utils_tests" to the command if not needed to pack server or tools, for example, the dependencies are static linked.
- - uses: "./.github/actions/build_pegasus"
- - uses: "./.github/actions/upload_artifact"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
+ - name: Upload artifact
+ uses: "./.github/actions/upload_artifact"
test_with_jemalloc:
name: Test with jemallc
@@ -347,8 +367,10 @@ jobs:
options: --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/download_artifact"
- - uses: "./.github/actions/run_server_tests"
+ - name: Download artifact
+ uses: "./.github/actions/download_artifact"
+ - name: Run server tests
+ uses: "./.github/actions/run_server_tests"
build_pegasus_on_macos:
name: macOS
diff --git a/.github/workflows/lint_and_test_java-client.yml b/.github/workflows/lint_and_test_java-client.yml
index dfe747bd8..119877d0e 100644
--- a/.github/workflows/lint_and_test_java-client.yml
+++ b/.github/workflows/lint_and_test_java-client.yml
@@ -59,9 +59,12 @@ jobs:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
- - uses: "./.github/actions/build_pegasus"
- - uses: "./.github/actions/upload_artifact"
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
+ - name: Upload artifact
+ uses: "./.github/actions/upload_artifact"
test_java_client:
name: Test Java client
@@ -86,7 +89,8 @@ jobs:
- uses: actions/setup-java@v1
with:
java-version: ${{ matrix.java }}
- - uses: "./.github/actions/download_artifact"
+ - name: Download artifact
+ uses: "./.github/actions/download_artifact"
- name: Start Pegasus cluster
run: |
apt-get update
diff --git a/.github/workflows/regular-build.yml b/.github/workflows/regular-build.yml
index f536479be..a8f438621 100644
--- a/.github/workflows/regular-build.yml
+++ b/.github/workflows/regular-build.yml
@@ -72,30 +72,11 @@ jobs:
working-directory: /root/incubator-pegasus
steps:
- name: Clone Apache Pegasus Source
- working-directory: /root
- run: |
- git clone -b ${{ github.ref_name }} --depth=1 https://github.com/apache/incubator-pegasus.git
- - name: Free Disk Space (Ubuntu)
- run: |
- .github/workflows/free_disk_space.sh
+ uses: actions/checkout@v3
- name: Unpack prebuilt third-parties
- run: |
- rm -f /root/thirdparties-src.zip
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- rm -rf ./thirdparty/hadoop-bin/share/doc
- rm -rf ./thirdparty/zookeeper-bin/docs
- - name: Compilation Pegasus
- run: |
- ./run.sh build --test --compiler ${{ matrix.compiler }} --skip_thirdparty -j $(nproc)
- - name: Clear Build Files
- run: |
- find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- - name: Packaging Server
- run: |
- ./run.sh pack_server
- rm -rf pegasus-server-*
+ uses: "./.github/actions/unpack_prebuilt_thirdparties"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
build_and_lint_go:
name: Build and Lint Golang
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org