You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nd...@apache.org on 2026/01/15 14:16:00 UTC
(hbase) branch branch-3 updated: HBASE-29787 Run Yetus General Checks as a GitHub Action (#7630)
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 8a73446d065 HBASE-29787 Run Yetus General Checks as a GitHub Action (#7630)
8a73446d065 is described below
commit 8a73446d0659c6fa1827f6cac99733c718183688
Author: Nick Dimiduk <nd...@apache.org>
AuthorDate: Thu Jan 15 15:15:54 2026 +0100
HBASE-29787 Run Yetus General Checks as a GitHub Action (#7630)
Signed-off-by: Duo Zhang <zh...@apache.org>
Signed-off-by: Peter Somogyi <pe...@apache.org>
---
.github/workflows/yetus-general-check.yml | 142 ++++++++++++++++++++++++++
dev-support/gh_hide_old_comments.sh | 14 ++-
dev-support/hbase-personality.sh | 34 +++++-
dev-support/jenkins_precommit_github_yetus.sh | 7 +-
4 files changed, 191 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/yetus-general-check.yml b/.github/workflows/yetus-general-check.yml
new file mode 100644
index 00000000000..664ef27624e
--- /dev/null
+++ b/.github/workflows/yetus-general-check.yml
@@ -0,0 +1,142 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# yamllint disable rule:line-length
+---
+name: Yetus General Check
+
+"on":
+ pull_request:
+ types: [opened, synchronize, reopened]
+
+permissions:
+ checks: write
+ contents: read
+ pull-requests: write
+ statuses: write
+
+jobs:
+ general-check:
+ runs-on: ubuntu-latest
+ timeout-minutes: 600
+
+ env:
+ YETUS_VERSION: '0.15.0'
+
+ steps:
+ - name: Checkout HBase
+ uses: actions/checkout@v4
+ with:
+ path: src
+ fetch-depth: 0
+
+ - name: Set up JDK 17
+ uses: actions/setup-java@v4
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+
+ - name: Maven cache
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2
+ key: hbase-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ hbase-m2-
+
+ - name: Download Yetus
+ run: |
+ mkdir -p yetus
+ cd yetus
+ bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
+ --keys 'https://downloads.apache.org/yetus/KEYS' \
+ --verify-tar-gz \
+ ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \
+ yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
+ tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
+ rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
+
+ - name: Run Yetus General Check
+ env:
+ ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump"
+ DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile"
+ GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+ GITHUB_USER: ${{ github.actor }}
+ PATCHDIR: "${{ github.workspace }}/yetus-general-check/output"
+ PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit"
+ SET_JAVA_HOME: "/usr/lib/jvm/java-17"
+ SOURCEDIR: "${{ github.workspace }}/src"
+ TESTS_FILTER: "checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop"
+ YETUSDIR: "${{ github.workspace }}/yetus"
+ AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc"
+ BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt"
+ BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt"
+ EXCLUDE_TESTS_URL: "https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref }}/lastSuccessfulBuild/artifact/output/excludes"
+ BUILD_THREAD: "4"
+ SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C"
+ SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C"
+ BRANCH_NAME: "${{ github.base_ref }}"
+ DEBUG: 'true'
+ run: |
+ cd "${{ github.workspace }}"
+ bash src/dev-support/jenkins_precommit_github_yetus.sh
+
+ - name: Hide Old Comments
+ if: always()
+ env:
+ DEBUG: 'true'
+ run: |
+ # github-actions[bot] has a known user ID: 41898282
+ # Verify: curl -s "https://api.github.com/users/github-actions%5Bbot%5D" | jq .id
+ export DEBUG="${DEBUG:-false}"
+ export GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}"
+ export BUILD_BOT_USER_ID="41898282"
+ export JOB_NAME="HBase-PreCommit-GH-Actions-PR"
+ export REPO="${{ github.repository }}"
+ bash src/dev-support/gh_hide_old_comments.sh ${{ github.event.pull_request.number }} || true
+
+ - name: Create Nightlies Directory Structure
+ # this version of rsync doesn't support --mkdirs so we do this step manually
+ if: always()
+ run: |
+ eval $(ssh-agent -s)
+ ssh-add - <<< "${{ secrets.NIGHTLIES_RSYNC_KEY }}"
+ ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} \
+ -o StrictHostKeyChecking=no \
+ ${{ secrets.NIGHTLIES_RSYNC_USER }}@${{ secrets.NIGHTLIES_RSYNC_HOST }} \
+ "mkdir -p ${{ secrets.NIGHTLIES_RSYNC_PATH }}/hbase/HBase-PreCommit-GH-Actions-PR/PR-${{ github.event.pull_request.number }}/${{ github.run_number }}"
+ ssh-agent -k
+
+ - name: Publish to Nightlies
+ if: always()
+ uses: burnett01/rsync-deployments@0dc935cdecc5f5e571865e60d2a6cdc673704823
+ with:
+ switches: -avzr --no-o
+ path: yetus-general-check
+ remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/hbase/HBase-PreCommit-GH-Actions-PR/PR-${{ github.event.pull_request.number }}/${{ github.run_number }}/
+ remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
+ remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
+ remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
+ remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
+
+ - name: Publish Test Results
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: yetus-general-check-output
+ path: ${{ github.workspace }}/yetus-general-check/output
+ retention-days: 7
diff --git a/dev-support/gh_hide_old_comments.sh b/dev-support/gh_hide_old_comments.sh
index 61217cfa241..1e96918db97 100755
--- a/dev-support/gh_hide_old_comments.sh
+++ b/dev-support/gh_hide_old_comments.sh
@@ -104,11 +104,16 @@ function identify_most_recent_build_number {
local pr="$1"
local comments_file="$2"
local jq_filter
+ local url_pattern="${JOB_NAME}/job/PR-${pr}/(?<buildnum>[0-9]+)/"
+ # GitHub Actions URLs don't have /job/ in them
+ if [[ "${JOB_NAME}" == *"GH-Actions"* ]]; then
+ url_pattern="${JOB_NAME}/PR-${pr}/(?<buildnum>[0-9]+)/"
+ fi
read -r -d '' jq_filter << EOF || :
.[] \
| select(.user.id == ${BUILD_BOT_USER_ID}) \
| .body \
-| capture("${JOB_NAME}/job/PR-${pr}/(?<buildnum>[0-9]+)/") \
+| capture("${url_pattern}") \
| .buildnum
EOF
@@ -122,10 +127,15 @@ function identify_old_comment_ids {
local comments_file="$2"
local most_recent_build_number="$3"
local jq_filter
+ local url_pattern="${JOB_NAME}/job/PR-${pr}/(?<buildnum>[0-9]+)/"
+ # GitHub Actions URLs don't have /job/ in them
+ if [[ "${JOB_NAME}" == *"GH-Actions"* ]]; then
+ url_pattern="${JOB_NAME}/PR-${pr}/(?<buildnum>[0-9]+)/"
+ fi
read -r -d '' jq_filter << EOF || :
.[] \
| select(.user.id == ${BUILD_BOT_USER_ID}) \
-| { node_id, buildnum: (.body | capture("${JOB_NAME}/job/PR-${pr}/(?<buildnum>[0-9]+)/") | .buildnum | tonumber) } \
+| { node_id, buildnum: (.body | capture("${url_pattern}") | .buildnum | tonumber) } \
| select(.buildnum < (${most_recent_build_number} | tonumber)) \
| .node_id
EOF
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index 9a5d34cc213..da19e1b6924 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -88,6 +88,20 @@ function personality_globals
# See HBASE-19902 for how we arrived at 20g.
#shellcheck disable=SC2034
DOCKERMEMLIMIT=20g
+
+ # Override BUILD_URL for GitHub Actions to point to nightlies
+ # instead of the GitHub Actions run page
+ # The robot sets BUILD_URL to the GHA run page, but we want nightlies URLs
+ if [[ "${GITHUB_ACTIONS}" == "true" ]] && [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
+ local pr_number
+ # GITHUB_REF is a standard GitHub Actions environment variable
+ # shellcheck disable=SC2153
+ pr_number=$(echo "${GITHUB_REF}" | cut -f3 -d/)
+ #shellcheck disable=SC2034
+ BUILD_URL="https://nightlies.apache.org/hbase/HBase-PreCommit-GH-Actions-PR/PR-${pr_number}/${GITHUB_RUN_NUMBER}/"
+ #shellcheck disable=SC2034
+ BUILD_URL_ARTIFACTS="yetus-general-check/output"
+ fi
}
## @description Parse extra arguments required by personalities, if any.
@@ -835,6 +849,19 @@ function hbaseanti_patchfile
add_test_type spotless
+## @description Build artifact URL for GitHub Actions
+## @audience private
+## @stability evolving
+## @replaceable no
+if ! declare -f githubactions_artifact_url >/dev/null; then
+ function githubactions_artifact_url
+ {
+ if [[ -n "${BUILD_URL}" ]] && [[ -n "${BUILD_URL_ARTIFACTS}" ]]; then
+ echo "${BUILD_URL}${BUILD_URL_ARTIFACTS}"
+ fi
+ }
+fi
+
## @description spotless file filter
## @audience private
## @stability evolving
@@ -869,12 +896,13 @@ function spotless_rebuild
count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
- add_vote_table -1 spotless "${repostatus} has ${count} errors when running spotless:check, run spotless:apply to fix."
- add_footer_table spotless "@@BASE@@/${repostatus}-spotless.txt"
+ add_vote_table_v2 -1 spotless \
+ "@@BASE@@/${repostatus}-spotless.txt" \
+ "${repostatus} has ${count} errors when running spotless:check, run spotless:apply to fix."
return 1
fi
- add_vote_table +1 spotless "${repostatus} has no errors when running spotless:check."
+ add_vote_table_v2 +1 spotless "" "${repostatus} has no errors when running spotless:check."
return 0
}
diff --git a/dev-support/jenkins_precommit_github_yetus.sh b/dev-support/jenkins_precommit_github_yetus.sh
index 8604d96760d..685ae3f186b 100755
--- a/dev-support/jenkins_precommit_github_yetus.sh
+++ b/dev-support/jenkins_precommit_github_yetus.sh
@@ -31,7 +31,6 @@ declare -i missing_env=0
declare -a required_envs=(
# these ENV variables define the required API with Jenkinsfile_GitHub
"ARCHIVE_PATTERN_LIST"
- "BUILD_URL_ARTIFACTS"
"DOCKERFILE"
"GITHUB_PASSWORD"
"GITHUB_USER"
@@ -53,6 +52,12 @@ for required_env in "${required_envs[@]}"; do
fi
done
+# BUILD_URL_ARTIFACTS is required for Jenkins but set in personality for GitHub Actions
+if [[ "${GITHUB_ACTIONS}" != "true" ]] && [[ -z "${BUILD_URL_ARTIFACTS}" ]]; then
+ echo "[ERROR] Required environment variable 'BUILD_URL_ARTIFACTS' is not set."
+ missing_env=${missing_env}+1
+fi
+
if [ ${missing_env} -gt 0 ]; then
echo "[ERROR] Please set the required environment variables before invoking. If this error is " \
"on Jenkins, then please file a JIRA about the error."