You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by la...@apache.org on 2023/12/22 08:43:07 UTC
(incubator-pegasus) branch ci_refactor_more created (now 5a37c6a49)
This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a change to branch ci_refactor_more
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
at 5a37c6a49 chore(CI): Improve github actions (2nd)
This branch includes the following new commits:
new 5a37c6a49 chore(CI): Improve github actions (2nd)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org
(incubator-pegasus) 01/01: chore(CI): Improve github actions (2nd)
Posted by la...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch ci_refactor_more
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
commit 5a37c6a49d30e4cbf1819bd56f2e284265f4c336
Author: Yingchun Lai <la...@apache.org>
AuthorDate: Fri Dec 22 16:42:45 2023 +0800
chore(CI): Improve github actions (2nd)
---
.github/actions/download_artifact/action.yaml | 5 +-
.../rebuild_thirdparty_if_needed/action.yaml | 9 +--
.../action.yaml | 18 ++----
.github/workflows/lint_and_test_cpp.yaml | 66 ++++++++++++++--------
.github/workflows/lint_and_test_java-client.yml | 12 ++--
.github/workflows/regular-build.yml | 27 ++-------
6 files changed, 64 insertions(+), 73 deletions(-)
diff --git a/.github/actions/download_artifact/action.yaml b/.github/actions/download_artifact/action.yaml
index 35ddb8ae0..29d68973c 100644
--- a/.github/actions/download_artifact/action.yaml
+++ b/.github/actions/download_artifact/action.yaml
@@ -20,10 +20,7 @@ runs:
using: composite
steps:
- name: Unpack prebuilt third-parties
- run: |
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- shell: bash
+ uses: ".github/actions/unpack_prebuilt_thirdparties/action.yaml"
- name: Download tarball
uses: actions/download-artifact@v3
with:
diff --git a/.github/actions/rebuild_thirdparty_if_needed/action.yaml b/.github/actions/rebuild_thirdparty_if_needed/action.yaml
index ff48fff29..035ae47d5 100644
--- a/.github/actions/rebuild_thirdparty_if_needed/action.yaml
+++ b/.github/actions/rebuild_thirdparty_if_needed/action.yaml
@@ -30,14 +30,7 @@ runs:
- 'thirdparty/**'
- name: Unpack prebuilt third-parties
if: steps.changes.outputs.thirdparty == 'false'
- run: |
- rm -f /root/thirdparties-src.zip
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- rm -rf ./thirdparty/hadoop-bin/share/doc
- rm -rf ./thirdparty/zookeeper-bin/docs
- shell: bash
+ uses: ".github/actions/unpack_prebuilt_thirdparties/action.yaml"
- name: Rebuild third-parties
if: steps.changes.outputs.thirdparty == 'true'
working-directory: thirdparty
diff --git a/.github/actions/download_artifact/action.yaml b/.github/actions/unpack_prebuilt_thirdparties/action.yaml
similarity index 71%
copy from .github/actions/download_artifact/action.yaml
copy to .github/actions/unpack_prebuilt_thirdparties/action.yaml
index 35ddb8ae0..0e1b2cdaf 100644
--- a/.github/actions/download_artifact/action.yaml
+++ b/.github/actions/unpack_prebuilt_thirdparties/action.yaml
@@ -15,22 +15,16 @@
# specific language governing permissions and limitations
# under the License.
-name: Download artifact
+name: Unpack prebuilt third-parties
runs:
using: composite
steps:
- - name: Unpack prebuilt third-parties
+ - name: Unpack
run: |
+ rm -f /root/thirdparties-src.zip
unzip /root/thirdparties-bin.zip -d ./thirdparty
rm -f /root/thirdparties-bin.zip
- shell: bash
- - name: Download tarball
- uses: actions/download-artifact@v3
- with:
- name: ${{ env.ARTIFACT_NAME }}_artifact_${{ github.sha }}
- path: .
- - name: Unpack tarball
- run: |
- tar -zxvf ${ARTIFACT_NAME}_builder.tar
- rm -f ${ARTIFACT_NAME}_builder.tar
+ find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
+ rm -rf ./thirdparty/hadoop-bin/share/doc
+ rm -rf ./thirdparty/zookeeper-bin/docs
shell: bash
diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml
index f90a18074..48abc8e19 100644
--- a/.github/workflows/lint_and_test_cpp.yaml
+++ b/.github/workflows/lint_and_test_cpp.yaml
@@ -72,7 +72,8 @@ jobs:
- name: Free Disk Space (Ubuntu)
run: |
.github/workflows/free_disk_space.sh
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
- name: Build IWYU binary
run: |
mkdir iwyu && cd iwyu && git clone https://github.com/include-what-you-use/include-what-you-use.git
@@ -97,10 +98,14 @@ jobs:
container:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
steps:
- - uses: actions/checkout@v3
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
- - uses: "./.github/actions/build_pegasus"
- - uses: "./.github/actions/upload_artifact"
+ - name: Clone code
+ uses: actions/checkout@v3
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
+ - name: Upload artifact
+ uses: "./.github/actions/upload_artifact"
test_Release:
name: Test Release
@@ -157,8 +162,10 @@ jobs:
options: --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/download_artifact"
- - uses: "./.github/actions/run_server_tests"
+ - name: Download artifact
+ uses: "./.github/actions/download_artifact"
+ - name: Run server tests
+ uses: "./.github/actions/run_server_tests"
build_ASAN:
name: Build ASAN
@@ -172,9 +179,12 @@ jobs:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
- - uses: "./.github/actions/build_pegasus"
- - uses: "./.github/actions/upload_artifact"
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
+ - name: Upload artifact
+ uses: "./.github/actions/upload_artifact"
test_ASAN:
name: Test ASAN
@@ -233,8 +243,10 @@ jobs:
options: --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/download_artifact"
- - uses: "./.github/actions/run_server_tests"
+ - name: Download artifact
+ uses: "./.github/actions/download_artifact"
+ - name: Run server tests
+ uses: "./.github/actions/run_server_tests"
# TODO(yingchun): Build and test UBSAN version would cost a very long time, we will run these tests
# when we are going to release a stable version. So we disable them in regular CI
@@ -251,9 +263,12 @@ jobs:
# image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
# steps:
# - uses: actions/checkout@v3
-# - uses: "./.github/actions/rebuild_thirdparty_if_needed"
-# - uses: "./.github/actions/build_pegasus"
-# - uses: "./.github/actions/upload_artifact"
+# - name: Rebuild thirdparty if needed
+# uses: "./.github/actions/rebuild_thirdparty_if_needed"
+# - name: Build Pegasus
+# uses: "./.github/actions/build_pegasus"
+# - name: Upload artifact
+# uses: "./.github/actions/upload_artifact"
#
# test_UBSAN:
# name: Test UBSAN
@@ -308,8 +323,10 @@ jobs:
# options: --cap-add=SYS_PTRACE
# steps:
# - uses: actions/checkout@v3
-# - uses: "./.github/actions/download_artifact"
-# - uses: "./.github/actions/run_server_tests"
+# - name: Download artifact
+# uses: "./.github/actions/download_artifact"
+# - name: Run server tests
+# uses: "./.github/actions/run_server_tests"
build_with_jemalloc:
name: Build with jemalloc
@@ -323,10 +340,13 @@ jobs:
image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }}
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
# TODO(yingchun): Append "-m dsn_utils_tests" to the command if not needed to pack server or tools, for example, the dependencies are static linked.
- - uses: "./.github/actions/build_pegasus"
- - uses: "./.github/actions/upload_artifact"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
+ - name: Upload artifact
+ uses: "./.github/actions/upload_artifact"
test_with_jemalloc:
name: Test with jemallc
@@ -344,8 +364,10 @@ jobs:
options: --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/download_artifact"
- - uses: "./.github/actions/run_server_tests"
+ - name: Download artifact
+ uses: "./.github/actions/download_artifact"
+ - name: Run server tests
+ uses: "./.github/actions/run_server_tests"
build_pegasus_on_macos:
name: macOS
diff --git a/.github/workflows/lint_and_test_java-client.yml b/.github/workflows/lint_and_test_java-client.yml
index dfe747bd8..119877d0e 100644
--- a/.github/workflows/lint_and_test_java-client.yml
+++ b/.github/workflows/lint_and_test_java-client.yml
@@ -59,9 +59,12 @@ jobs:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
steps:
- uses: actions/checkout@v3
- - uses: "./.github/actions/rebuild_thirdparty_if_needed"
- - uses: "./.github/actions/build_pegasus"
- - uses: "./.github/actions/upload_artifact"
+ - name: Rebuild thirdparty if needed
+ uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
+ - name: Upload artifact
+ uses: "./.github/actions/upload_artifact"
test_java_client:
name: Test Java client
@@ -86,7 +89,8 @@ jobs:
- uses: actions/setup-java@v1
with:
java-version: ${{ matrix.java }}
- - uses: "./.github/actions/download_artifact"
+ - name: Download artifact
+ uses: "./.github/actions/download_artifact"
- name: Start Pegasus cluster
run: |
apt-get update
diff --git a/.github/workflows/regular-build.yml b/.github/workflows/regular-build.yml
index f536479be..da20c54d8 100644
--- a/.github/workflows/regular-build.yml
+++ b/.github/workflows/regular-build.yml
@@ -72,30 +72,11 @@ jobs:
working-directory: /root/incubator-pegasus
steps:
- name: Clone Apache Pegasus Source
- working-directory: /root
- run: |
- git clone -b ${{ github.ref_name }} --depth=1 https://github.com/apache/incubator-pegasus.git
- - name: Free Disk Space (Ubuntu)
- run: |
- .github/workflows/free_disk_space.sh
+ uses: actions/checkout@v3
- name: Unpack prebuilt third-parties
- run: |
- rm -f /root/thirdparties-src.zip
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- rm -rf ./thirdparty/hadoop-bin/share/doc
- rm -rf ./thirdparty/zookeeper-bin/docs
- - name: Compilation Pegasus
- run: |
- ./run.sh build --test --compiler ${{ matrix.compiler }} --skip_thirdparty -j $(nproc)
- - name: Clear Build Files
- run: |
- find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- - name: Packaging Server
- run: |
- ./run.sh pack_server
- rm -rf pegasus-server-*
+ uses: ".github/actions/unpack_prebuilt_thirdparties/action.yaml"
+ - name: Build Pegasus
+ uses: "./.github/actions/build_pegasus"
build_and_lint_go:
name: Build and Lint Golang
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org