You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by la...@apache.org on 2024/01/02 03:02:04 UTC

(incubator-pegasus) branch master updated: chore(CI): Improve github actions (2nd) (#1812)

This is an automated email from the ASF dual-hosted git repository.

laiyingchun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git


The following commit(s) were added to refs/heads/master by this push:
     new 394617b77 chore(CI): Improve github actions (2nd) (#1812)
394617b77 is described below

commit 394617b778538c1842f2423f48dd763bfda09d85
Author: Yingchun Lai <la...@apache.org>
AuthorDate: Tue Jan 2 11:01:59 2024 +0800

    chore(CI): Improve github actions (2nd) (#1812)
    
    - Make the "Unpack prebuilt third-parties" as a independent job
    - Add names for the jobs which calling the action yamls
---
 .github/actions/download_artifact/action.yaml      |  5 +-
 .../rebuild_thirdparty_if_needed/action.yaml       |  9 +--
 .../action.yaml                                    | 18 ++----
 .github/workflows/lint_and_test_cpp.yaml           | 66 ++++++++++++++--------
 .github/workflows/lint_and_test_java-client.yml    | 12 ++--
 .github/workflows/regular-build.yml                | 27 ++-------
 6 files changed, 64 insertions(+), 73 deletions(-)

diff --git a/.github/actions/download_artifact/action.yaml b/.github/actions/download_artifact/action.yaml
index 35ddb8ae0..07cf2d476 100644
--- a/.github/actions/download_artifact/action.yaml
+++ b/.github/actions/download_artifact/action.yaml
@@ -20,10 +20,7 @@ runs:
   using: composite
   steps:
     - name: Unpack prebuilt third-parties
-      run: |
-        unzip /root/thirdparties-bin.zip -d ./thirdparty
-        rm -f /root/thirdparties-bin.zip
-      shell: bash
+      uses: "./.github/actions/unpack_prebuilt_thirdparties"
     - name: Download tarball
       uses: actions/download-artifact@v3
       with:
diff --git a/.github/actions/rebuild_thirdparty_if_needed/action.yaml b/.github/actions/rebuild_thirdparty_if_needed/action.yaml
index ff48fff29..6e64f7db4 100644
--- a/.github/actions/rebuild_thirdparty_if_needed/action.yaml
+++ b/.github/actions/rebuild_thirdparty_if_needed/action.yaml
@@ -30,14 +30,7 @@ runs:
             - 'thirdparty/**'
     - name: Unpack prebuilt third-parties
       if: steps.changes.outputs.thirdparty == 'false'
-      run: |
-        rm -f /root/thirdparties-src.zip
-        unzip /root/thirdparties-bin.zip -d ./thirdparty
-        rm -f /root/thirdparties-bin.zip
-        find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-        rm -rf ./thirdparty/hadoop-bin/share/doc
-        rm -rf ./thirdparty/zookeeper-bin/docs
-      shell: bash
+      uses: "./.github/actions/unpack_prebuilt_thirdparties"
     - name: Rebuild third-parties
       if: steps.changes.outputs.thirdparty == 'true'
       working-directory: thirdparty
diff --git a/.github/actions/download_artifact/action.yaml b/.github/actions/unpack_prebuilt_thirdparties/action.yaml
similarity index 71%
copy from .github/actions/download_artifact/action.yaml
copy to .github/actions/unpack_prebuilt_thirdparties/action.yaml
index 35ddb8ae0..0e1b2cdaf 100644
--- a/.github/actions/download_artifact/action.yaml
+++ b/.github/actions/unpack_prebuilt_thirdparties/action.yaml
@@ -15,22 +15,16 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name: Download artifact
+name: Unpack prebuilt third-parties
 runs:
   using: composite
   steps:
-    - name: Unpack prebuilt third-parties
+    - name: Unpack
       run: |
+        rm -f /root/thirdparties-src.zip
         unzip /root/thirdparties-bin.zip -d ./thirdparty
         rm -f /root/thirdparties-bin.zip
-      shell: bash
-    - name: Download tarball
-      uses: actions/download-artifact@v3
-      with:
-        name: ${{ env.ARTIFACT_NAME }}_artifact_${{ github.sha }}
-        path: .
-    - name: Unpack tarball
-      run: |
-        tar -zxvf ${ARTIFACT_NAME}_builder.tar
-        rm -f ${ARTIFACT_NAME}_builder.tar
+        find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
+        rm -rf ./thirdparty/hadoop-bin/share/doc
+        rm -rf ./thirdparty/zookeeper-bin/docs
       shell: bash
diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml
index 324bf00c6..a17121972 100644
--- a/.github/workflows/lint_and_test_cpp.yaml
+++ b/.github/workflows/lint_and_test_cpp.yaml
@@ -72,7 +72,8 @@ jobs:
       - name: Free Disk Space (Ubuntu)
         run: |
           .github/workflows/free_disk_space.sh
-      - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+      - name: Rebuild thirdparty if needed
+        uses: "./.github/actions/rebuild_thirdparty_if_needed"
       - name: Build IWYU binary
         run: |
           mkdir iwyu && cd iwyu && git clone https://github.com/include-what-you-use/include-what-you-use.git
@@ -97,10 +98,14 @@ jobs:
     container:
       image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: "./.github/actions/rebuild_thirdparty_if_needed"
-      - uses: "./.github/actions/build_pegasus"
-      - uses: "./.github/actions/upload_artifact"
+      - name: Clone code
+        uses: actions/checkout@v3
+      - name: Rebuild thirdparty if needed
+        uses: "./.github/actions/rebuild_thirdparty_if_needed"
+      - name: Build Pegasus
+        uses: "./.github/actions/build_pegasus"
+      - name: Upload artifact
+        uses: "./.github/actions/upload_artifact"
 
   test_Release:
     name: Test Release
@@ -158,8 +163,10 @@ jobs:
       options: --cap-add=SYS_PTRACE
     steps:
       - uses: actions/checkout@v3
-      - uses: "./.github/actions/download_artifact"
-      - uses: "./.github/actions/run_server_tests"
+      - name: Download artifact
+        uses: "./.github/actions/download_artifact"
+      - name: Run server tests
+        uses: "./.github/actions/run_server_tests"
 
   build_ASAN:
     name: Build ASAN
@@ -173,9 +180,12 @@ jobs:
       image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
     steps:
       - uses: actions/checkout@v3
-      - uses: "./.github/actions/rebuild_thirdparty_if_needed"
-      - uses: "./.github/actions/build_pegasus"
-      - uses: "./.github/actions/upload_artifact"
+      - name: Rebuild thirdparty if needed
+        uses: "./.github/actions/rebuild_thirdparty_if_needed"
+      - name: Build Pegasus
+        uses: "./.github/actions/build_pegasus"
+      - name: Upload artifact
+        uses: "./.github/actions/upload_artifact"
 
   test_ASAN:
     name: Test ASAN
@@ -235,8 +245,10 @@ jobs:
       options: --cap-add=SYS_PTRACE
     steps:
       - uses: actions/checkout@v3
-      - uses: "./.github/actions/download_artifact"
-      - uses: "./.github/actions/run_server_tests"
+      - name: Download artifact
+        uses: "./.github/actions/download_artifact"
+      - name: Run server tests
+        uses: "./.github/actions/run_server_tests"
 
 # TODO(yingchun): Build and test UBSAN version would cost a very long time, we will run these tests
 #                 when we are going to release a stable version. So we disable them in regular CI
@@ -253,9 +265,12 @@ jobs:
 #      image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
 #    steps:
 #      - uses: actions/checkout@v3
-#      - uses: "./.github/actions/rebuild_thirdparty_if_needed"
-#      - uses: "./.github/actions/build_pegasus"
-#      - uses: "./.github/actions/upload_artifact"
+#      - name: Rebuild thirdparty if needed
+#        uses: "./.github/actions/rebuild_thirdparty_if_needed"
+#      - name: Build Pegasus
+#        uses: "./.github/actions/build_pegasus"
+#      - name: Upload artifact
+#        uses: "./.github/actions/upload_artifact"
 #
 #  test_UBSAN:
 #    name: Test UBSAN
@@ -311,8 +326,10 @@ jobs:
 #      options: --cap-add=SYS_PTRACE
 #    steps:
 #      - uses: actions/checkout@v3
-#      - uses: "./.github/actions/download_artifact"
-#      - uses: "./.github/actions/run_server_tests"
+#      - name: Download artifact
+#        uses: "./.github/actions/download_artifact"
+#      - name: Run server tests
+#        uses: "./.github/actions/run_server_tests"
 
   build_with_jemalloc:
     name: Build with jemalloc
@@ -326,10 +343,13 @@ jobs:
       image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }}
     steps:
       - uses: actions/checkout@v3
-      - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+      - name: Rebuild thirdparty if needed
+        uses: "./.github/actions/rebuild_thirdparty_if_needed"
       # TODO(yingchun): Append "-m dsn_utils_tests" to the command if not needed to pack server or tools, for example, the dependencies are static linked.
-      - uses: "./.github/actions/build_pegasus"
-      - uses: "./.github/actions/upload_artifact"
+      - name: Build Pegasus
+        uses: "./.github/actions/build_pegasus"
+      - name: Upload artifact
+        uses: "./.github/actions/upload_artifact"
 
   test_with_jemalloc:
     name: Test with jemallc
@@ -347,8 +367,10 @@ jobs:
       options: --cap-add=SYS_PTRACE
     steps:
       - uses: actions/checkout@v3
-      - uses: "./.github/actions/download_artifact"
-      - uses: "./.github/actions/run_server_tests"
+      - name: Download artifact
+        uses: "./.github/actions/download_artifact"
+      - name: Run server tests
+        uses: "./.github/actions/run_server_tests"
 
   build_pegasus_on_macos:
     name: macOS
diff --git a/.github/workflows/lint_and_test_java-client.yml b/.github/workflows/lint_and_test_java-client.yml
index dfe747bd8..119877d0e 100644
--- a/.github/workflows/lint_and_test_java-client.yml
+++ b/.github/workflows/lint_and_test_java-client.yml
@@ -59,9 +59,12 @@ jobs:
       image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
     steps:
       - uses: actions/checkout@v3
-      - uses: "./.github/actions/rebuild_thirdparty_if_needed"
-      - uses: "./.github/actions/build_pegasus"
-      - uses: "./.github/actions/upload_artifact"
+      - name: Rebuild thirdparty if needed
+        uses: "./.github/actions/rebuild_thirdparty_if_needed"
+      - name: Build Pegasus
+        uses: "./.github/actions/build_pegasus"
+      - name: Upload artifact
+        uses: "./.github/actions/upload_artifact"
 
   test_java_client:
     name: Test Java client
@@ -86,7 +89,8 @@ jobs:
       - uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
-      - uses: "./.github/actions/download_artifact"
+      - name: Download artifact
+        uses: "./.github/actions/download_artifact"
       - name: Start Pegasus cluster
         run: |
           apt-get update
diff --git a/.github/workflows/regular-build.yml b/.github/workflows/regular-build.yml
index f536479be..a8f438621 100644
--- a/.github/workflows/regular-build.yml
+++ b/.github/workflows/regular-build.yml
@@ -72,30 +72,11 @@ jobs:
         working-directory: /root/incubator-pegasus
     steps:
       - name: Clone Apache Pegasus Source
-        working-directory: /root
-        run: |
-          git clone -b ${{ github.ref_name }} --depth=1 https://github.com/apache/incubator-pegasus.git
-      - name: Free Disk Space (Ubuntu)
-        run: |
-          .github/workflows/free_disk_space.sh
+        uses: actions/checkout@v3
       - name: Unpack prebuilt third-parties
-        run: |
-          rm -f /root/thirdparties-src.zip
-          unzip /root/thirdparties-bin.zip -d ./thirdparty
-          rm -f /root/thirdparties-bin.zip
-          find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-          rm -rf ./thirdparty/hadoop-bin/share/doc
-          rm -rf ./thirdparty/zookeeper-bin/docs
-      - name: Compilation Pegasus
-        run: |
-          ./run.sh build --test --compiler ${{ matrix.compiler }} --skip_thirdparty -j $(nproc)
-      - name: Clear Build Files
-        run: |
-          find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-      - name: Packaging Server
-        run: |
-          ./run.sh pack_server
-          rm -rf pegasus-server-*
+        uses: "./.github/actions/unpack_prebuilt_thirdparties"
+      - name: Build Pegasus
+        uses: "./.github/actions/build_pegasus"
 
   build_and_lint_go:
     name: Build and Lint Golang


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org