You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/04/15 13:23:46 UTC
(pulsar) 03/07: [improve][ci] Cleanup tune-runner-vm and clean-disk actions (#25444)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit d8170d535c54cad0c3edc7b94cf2585e4bdce15a
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Tue Mar 31 23:20:40 2026 +0300
[improve][ci] Cleanup tune-runner-vm and clean-disk actions (#25444)
(cherry picked from commit 396290adabdf85f905add66b03f406b1c14cd20b)
---
.github/actions/clean-disk/action.yml | 116 ++++++++++++++++++++++--------
.github/actions/tune-runner-vm/action.yml | 50 +++++--------
2 files changed, 102 insertions(+), 64 deletions(-)
diff --git a/.github/actions/clean-disk/action.yml b/.github/actions/clean-disk/action.yml
index e67ce59a08d..fe5fa9e7cc9 100644
--- a/.github/actions/clean-disk/action.yml
+++ b/.github/actions/clean-disk/action.yml
@@ -23,44 +23,98 @@ inputs:
mode:
description: "Use 'full' to clean as much as possible"
required: false
+ clean-threshold-gb:
+ description: "Skip cleaning if available disk space exceeds this threshold (in GB)"
+ required: false
+ default: '20'
+ full-clean-threshold-gb:
+ description: "In 'full' mode, skip additional cleaning steps if available disk space exceeds this threshold (in GB)"
+ required: false
+ default: '40'
runs:
using: composite
steps:
- - run: |
+ - shell: bash
+ env:
+ CLEAN_MODE: ${{ inputs.mode }}
+ CLEAN_THRESHOLD_GB: ${{ inputs.clean-threshold-gb }}
+ FULL_CLEAN_THRESHOLD_GB: ${{ inputs.full-clean-threshold-gb }}
+ run: |
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
- directories=(/usr/local/lib/android /opt/ghc)
- if [[ "${{ inputs.mode }}" == "full" ]]; then
- # remove these directories only when mode is 'full'
- directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL)
+
+ available_gb() {
+ df --output=avail -BG / | tail -1 | tr -d ' G'
+ }
+
+ show_disk() {
+ if mountpoint -q /mnt; then df -BM / /mnt; else df -BM /; fi
+ }
+
+ remove_dir() {
+ local directory=$1
+ if [ -d "$directory" ]; then
+ echo "::group::Removing $directory"
+ local emptydir=/tmp/empty$$/
+ mkdir -p "$emptydir"
+ # fast way to delete a lot of files on linux
+ time sudo eatmydata rsync -a --delete "$emptydir" "${directory}/"
+ time sudo eatmydata rm -rf "${directory}"
+ rmdir "$emptydir" 2>/dev/null || true
+ time show_disk
+ echo "::endgroup::"
+ fi
+ }
+
+ # Remove directories one at a time, stopping when threshold is met
+ clean_dirs() {
+ local threshold=$1
+ shift
+ for directory in "$@"; do
+ if (( $(available_gb) >= threshold )); then
+ return
+ fi
+ remove_dir "$directory"
+ done
+ }
+
+ if [[ "$CLEAN_MODE" == "full" ]]; then
+ threshold=$FULL_CLEAN_THRESHOLD_GB
+ else
+ threshold=$CLEAN_THRESHOLD_GB
fi
- emptydir=/tmp/empty$$/
- mkdir $emptydir
- echo "::group::Available diskspace"
- time df -BM / /mnt
+
+ echo "::group::Available diskspace (threshold: ${threshold}GB)"
+ time show_disk
echo "::endgroup::"
- for directory in "${directories[@]}"; do
- echo "::group::Removing $directory"
- # fast way to delete a lot of files on linux
- time sudo eatmydata rsync -a --delete $emptydir ${directory}/
- time sudo eatmydata rm -rf ${directory}
- time df -BM / /mnt
- echo "::endgroup::"
- done
- if [[ "${{ inputs.mode }}" == "full" ]]; then
- echo "::group::Moving /var/lib/docker to /mnt/docker"
- sudo systemctl stop docker
- echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json
- sudo mv /var/lib/docker /mnt/docker
- sudo systemctl start docker
- time df -BM / /mnt
+
+ if (( $(available_gb) >= threshold )); then
+ echo "Available disk space meets threshold, skipping cleanup."
+ exit 0
+ fi
+
+ clean_dirs "$threshold" /usr/local/lib/android /opt/ghc
+
+ if [[ "$CLEAN_MODE" == "full" ]]; then
+ clean_dirs "$threshold" /usr/share/dotnet /opt/hostedtoolcache/CodeQL
+ if (( $(available_gb) < threshold )) && mountpoint -q /mnt; then
+ echo "::group::Moving /var/lib/docker to /mnt/docker"
+ sudo systemctl stop docker
+ echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json
+ sudo mv /var/lib/docker /mnt/docker
+ sudo systemctl start docker
+ time show_disk
+ echo "::endgroup::"
+ fi
+ fi
+
+ if (( $(available_gb) < threshold )); then
+ echo "::group::Cleaning apt state"
+ time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove"
+ time show_disk
echo "::endgroup::"
fi
- echo "::group::Cleaning apt state"
- time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove"
- time df -BM / /mnt
+
+ echo "::group::Available diskspace"
+ time show_disk
echo "::endgroup::"
fi
- echo "::group::Available diskspace"
- time df -BM / /mnt
- echo "::endgroup::"
- shell: bash
diff --git a/.github/actions/tune-runner-vm/action.yml b/.github/actions/tune-runner-vm/action.yml
index ab0f65767a6..d071f534dbb 100644
--- a/.github/actions/tune-runner-vm/action.yml
+++ b/.github/actions/tune-runner-vm/action.yml
@@ -22,7 +22,8 @@ description: tunes the GitHub Runner VM operation system
runs:
using: composite
steps:
- - run: |
+ - shell: bash
+ run: |
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
echo "::group::Configure and tune OS"
# Ensure that reverse lookups for current hostname are handled properly
@@ -33,49 +34,38 @@ runs:
# consumption is high.
# Set vm.swappiness=1 to avoid swapping and allow high RAM usage
echo 1 | sudo tee /proc/sys/vm/swappiness
- (
- shopt -s nullglob
- # Set swappiness to 1 for all cgroups and sub-groups
- for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do
- echo 1 | sudo tee $swappiness_file > /dev/null
- done
- ) || true
# use "madvise" Linux Transparent HugePages (THP) setting
# https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
# "madvise" is generally a better option than the default "always" setting
- # Based on Azul instructions from https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp
+ # recommendation from https://netflixtechblog.com/bending-pause-times-to-your-will-with-generational-zgc-256629c9386b
echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled
- echo defer+madvise | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
+ echo defer | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag
# tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
# commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
# nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true
- sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true
+ if mountpoint -q /mnt; then
+ sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true
+ fi
# disable discard/trim at device level since remount with nodiscard doesn't seem to be effective
# https://www.spinics.net/lists/linux-ide/msg52562.html
for i in /sys/block/sd*/queue/discard_max_bytes; do
echo 0 | sudo tee $i
done
- # disable any background jobs that run SSD discard/trim
- sudo systemctl disable fstrim.timer || true
- sudo systemctl stop fstrim.timer || true
- sudo systemctl disable fstrim.service || true
- sudo systemctl stop fstrim.service || true
+ # disable unnecessary timers
+ sudo systemctl stop fstrim.timer fstrim.service \
+ podman-auto-update.timer sysstat-collect.timer sysstat-summary.timer \
+ phpsessionclean.timer man-db.timer motd-news.timer \
+ dpkg-db-backup.timer e2scrub_all.timer \
+ update-notifier-download.timer update-notifier-motd.timer || true
- # stop php-fpm
- sudo systemctl stop php8.0-fpm.service || true
- sudo systemctl stop php7.4-fpm.service || true
- # stop mono-xsp4
- sudo systemctl disable mono-xsp4.service || true
- sudo systemctl stop mono-xsp4.service || true
- sudo killall mono || true
-
- # stop Azure Linux agent to save RAM
- sudo systemctl stop walinuxagent.service || true
+ # stop unnecessary services
+ sudo systemctl stop php8.3-fpm.service ModemManager.service \
+ multipathd.service udisks2.service walinuxagent.service || true
echo '::endgroup::'
@@ -87,10 +77,4 @@ runs:
echo "::group::Available diskspace"
df -BM
echo "::endgroup::"
- # show cggroup
- echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP"
- CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{ print $3 }')
- sudo cgget -a $CURRENT_CGGROUP || true
- echo '::endgroup::'
- fi
- shell: bash
+ fi
\ No newline at end of file