You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nd...@apache.org on 2026/01/15 14:16:00 UTC

(hbase) branch branch-3 updated: HBASE-29787 Run Yetus General Checks as a GitHub Action (#7630)

This is an automated email from the ASF dual-hosted git repository.

ndimiduk pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-3 by this push:
     new 8a73446d065 HBASE-29787 Run Yetus General Checks as a GitHub Action (#7630)
8a73446d065 is described below

commit 8a73446d0659c6fa1827f6cac99733c718183688
Author: Nick Dimiduk <nd...@apache.org>
AuthorDate: Thu Jan 15 15:15:54 2026 +0100

    HBASE-29787 Run Yetus General Checks as a GitHub Action (#7630)
    
    Signed-off-by: Duo Zhang <zh...@apache.org>
    Signed-off-by: Peter Somogyi <pe...@apache.org>
---
 .github/workflows/yetus-general-check.yml     | 142 ++++++++++++++++++++++++++
 dev-support/gh_hide_old_comments.sh           |  14 ++-
 dev-support/hbase-personality.sh              |  34 +++++-
 dev-support/jenkins_precommit_github_yetus.sh |   7 +-
 4 files changed, 191 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/yetus-general-check.yml b/.github/workflows/yetus-general-check.yml
new file mode 100644
index 00000000000..664ef27624e
--- /dev/null
+++ b/.github/workflows/yetus-general-check.yml
@@ -0,0 +1,142 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# yamllint disable rule:line-length
+---
+name: Yetus General Check
+
+"on":
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  checks: write
+  contents: read
+  pull-requests: write
+  statuses: write
+
+jobs:
+  general-check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 600
+
+    env:
+      YETUS_VERSION: '0.15.0'
+
+    steps:
+      - name: Checkout HBase
+        uses: actions/checkout@v4
+        with:
+          path: src
+          fetch-depth: 0
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+
+      - name: Maven cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2
+          key: hbase-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            hbase-m2-
+
+      - name: Download Yetus
+        run: |
+          mkdir -p yetus
+          cd yetus
+          bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
+            --keys 'https://downloads.apache.org/yetus/KEYS' \
+            --verify-tar-gz \
+            ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \
+            yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
+          tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
+          rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
+
+      - name: Run Yetus General Check
+        env:
+          ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump"
+          DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile"
+          GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_USER: ${{ github.actor }}
+          PATCHDIR: "${{ github.workspace }}/yetus-general-check/output"
+          PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit"
+          SET_JAVA_HOME: "/usr/lib/jvm/java-17"
+          SOURCEDIR: "${{ github.workspace }}/src"
+          TESTS_FILTER: "checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop"
+          YETUSDIR: "${{ github.workspace }}/yetus"
+          AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc"
+          BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt"
+          BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt"
+          EXCLUDE_TESTS_URL: "https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref }}/lastSuccessfulBuild/artifact/output/excludes"
+          BUILD_THREAD: "4"
+          SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C"
+          SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C"
+          BRANCH_NAME: "${{ github.base_ref }}"
+          DEBUG: 'true'
+        run: |
+          cd "${{ github.workspace }}"
+          bash src/dev-support/jenkins_precommit_github_yetus.sh
+
+      - name: Hide Old Comments
+        if: always()
+        env:
+          DEBUG: 'true'
+        run: |
+          # github-actions[bot] has a known user ID: 41898282
+          # Verify: curl -s "https://api.github.com/users/github-actions%5Bbot%5D" | jq .id
+          export DEBUG="${DEBUG:-false}"
+          export GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}"
+          export BUILD_BOT_USER_ID="41898282"
+          export JOB_NAME="HBase-PreCommit-GH-Actions-PR"
+          export REPO="${{ github.repository }}"
+          bash src/dev-support/gh_hide_old_comments.sh ${{ github.event.pull_request.number }} || true
+
+      - name: Create Nightlies Directory Structure
+        # this version of rsync doesn't support --mkdirs so we do this step manually
+        if: always()
+        run: |
+          eval $(ssh-agent -s)
+          ssh-add - <<< "${{ secrets.NIGHTLIES_RSYNC_KEY }}"
+          ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} \
+            -o StrictHostKeyChecking=no \
+            ${{ secrets.NIGHTLIES_RSYNC_USER }}@${{ secrets.NIGHTLIES_RSYNC_HOST }} \
+            "mkdir -p ${{ secrets.NIGHTLIES_RSYNC_PATH }}/hbase/HBase-PreCommit-GH-Actions-PR/PR-${{ github.event.pull_request.number }}/${{ github.run_number }}"
+          ssh-agent -k
+
+      - name: Publish to Nightlies
+        if: always()
+        uses: burnett01/rsync-deployments@0dc935cdecc5f5e571865e60d2a6cdc673704823
+        with:
+          switches: -avzr --no-o
+          path: yetus-general-check
+          remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/hbase/HBase-PreCommit-GH-Actions-PR/PR-${{ github.event.pull_request.number }}/${{ github.run_number }}/
+          remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
+          remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
+          remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
+          remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
+
+      - name: Publish Test Results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: yetus-general-check-output
+          path: ${{ github.workspace }}/yetus-general-check/output
+          retention-days: 7
diff --git a/dev-support/gh_hide_old_comments.sh b/dev-support/gh_hide_old_comments.sh
index 61217cfa241..1e96918db97 100755
--- a/dev-support/gh_hide_old_comments.sh
+++ b/dev-support/gh_hide_old_comments.sh
@@ -104,11 +104,16 @@ function identify_most_recent_build_number {
     local pr="$1"
     local comments_file="$2"
     local jq_filter
+    local url_pattern="${JOB_NAME}/job/PR-${pr}/(?<buildnum>[0-9]+)/"
+    # GitHub Actions URLs don't have /job/ in them
+    if [[ "${JOB_NAME}" == *"GH-Actions"* ]]; then
+        url_pattern="${JOB_NAME}/PR-${pr}/(?<buildnum>[0-9]+)/"
+    fi
     read -r -d '' jq_filter << EOF || :
 .[] \
 | select(.user.id == ${BUILD_BOT_USER_ID}) \
 | .body \
-| capture("${JOB_NAME}/job/PR-${pr}/(?<buildnum>[0-9]+)/") \
+| capture("${url_pattern}") \
 | .buildnum
 EOF
 
@@ -122,10 +127,15 @@ function identify_old_comment_ids {
     local comments_file="$2"
     local most_recent_build_number="$3"
     local jq_filter
+    local url_pattern="${JOB_NAME}/job/PR-${pr}/(?<buildnum>[0-9]+)/"
+    # GitHub Actions URLs don't have /job/ in them
+    if [[ "${JOB_NAME}" == *"GH-Actions"* ]]; then
+        url_pattern="${JOB_NAME}/PR-${pr}/(?<buildnum>[0-9]+)/"
+    fi
     read -r -d '' jq_filter << EOF || :
 .[] \
 | select(.user.id == ${BUILD_BOT_USER_ID}) \
-| { node_id, buildnum: (.body | capture("${JOB_NAME}/job/PR-${pr}/(?<buildnum>[0-9]+)/") | .buildnum | tonumber) } \
+| { node_id, buildnum: (.body | capture("${url_pattern}") | .buildnum | tonumber) } \
 | select(.buildnum < (${most_recent_build_number} | tonumber)) \
 | .node_id
 EOF
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index 9a5d34cc213..da19e1b6924 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -88,6 +88,20 @@ function personality_globals
   # See HBASE-19902 for how we arrived at 20g.
   #shellcheck disable=SC2034
   DOCKERMEMLIMIT=20g
+
+  # Override BUILD_URL for GitHub Actions to point to nightlies
+  # instead of the GitHub Actions run page
+  # The robot sets BUILD_URL to the GHA run page, but we want nightlies URLs
+  if [[ "${GITHUB_ACTIONS}" == "true" ]] && [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
+    local pr_number
+    # GITHUB_REF is a standard GitHub Actions environment variable
+    # shellcheck disable=SC2153
+    pr_number=$(echo "${GITHUB_REF}" | cut -f3 -d/)
+    #shellcheck disable=SC2034
+    BUILD_URL="https://nightlies.apache.org/hbase/HBase-PreCommit-GH-Actions-PR/PR-${pr_number}/${GITHUB_RUN_NUMBER}/"
+    #shellcheck disable=SC2034
+    BUILD_URL_ARTIFACTS="yetus-general-check/output"
+  fi
 }
 
 ## @description  Parse extra arguments required by personalities, if any.
@@ -835,6 +849,19 @@ function hbaseanti_patchfile
 
 add_test_type spotless
 
+## @description  Build artifact URL for GitHub Actions
+## @audience     private
+## @stability    evolving
+## @replaceable  no
+if ! declare -f githubactions_artifact_url >/dev/null; then
+  function githubactions_artifact_url
+  {
+    if [[ -n "${BUILD_URL}" ]] && [[ -n "${BUILD_URL_ARTIFACTS}" ]]; then
+      echo "${BUILD_URL}${BUILD_URL_ARTIFACTS}"
+    fi
+  }
+fi
+
 ## @description  spotless file filter
 ## @audience     private
 ## @stability    evolving
@@ -869,12 +896,13 @@ function spotless_rebuild
 
   count=$(${GREP} -c '\[ERROR\]' "${logfile}")
   if [[ ${count} -gt 0 ]]; then
-    add_vote_table -1 spotless "${repostatus} has ${count} errors when running spotless:check, run spotless:apply to fix."
-    add_footer_table spotless "@@BASE@@/${repostatus}-spotless.txt"
+    add_vote_table_v2 -1 spotless \
+      "@@BASE@@/${repostatus}-spotless.txt" \
+      "${repostatus} has ${count} errors when running spotless:check, run spotless:apply to fix."
     return 1
   fi
 
-  add_vote_table +1 spotless "${repostatus} has no errors when running spotless:check."
+  add_vote_table_v2 +1 spotless "" "${repostatus} has no errors when running spotless:check."
   return 0
 }
 
diff --git a/dev-support/jenkins_precommit_github_yetus.sh b/dev-support/jenkins_precommit_github_yetus.sh
index 8604d96760d..685ae3f186b 100755
--- a/dev-support/jenkins_precommit_github_yetus.sh
+++ b/dev-support/jenkins_precommit_github_yetus.sh
@@ -31,7 +31,6 @@ declare -i missing_env=0
 declare -a required_envs=(
   # these ENV variables define the required API with Jenkinsfile_GitHub
   "ARCHIVE_PATTERN_LIST"
-  "BUILD_URL_ARTIFACTS"
   "DOCKERFILE"
   "GITHUB_PASSWORD"
   "GITHUB_USER"
@@ -53,6 +52,12 @@ for required_env in "${required_envs[@]}"; do
   fi
 done
 
+# BUILD_URL_ARTIFACTS is required for Jenkins but set in personality for GitHub Actions
+if [[ "${GITHUB_ACTIONS}" != "true" ]] && [[ -z "${BUILD_URL_ARTIFACTS}" ]]; then
+  echo "[ERROR] Required environment variable 'BUILD_URL_ARTIFACTS' is not set."
+  missing_env=${missing_env}+1
+fi
+
 if [ ${missing_env} -gt 0 ]; then
   echo "[ERROR] Please set the required environment variables before invoking. If this error is " \
        "on Jenkins, then please file a JIRA about the error."