You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/10/14 18:24:50 UTC
(iggy) 01/01: perf(ci): avoid redundant Rust coverage builds
This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a commit to branch perf/rust-pre-merge-coverage
in repository https://gitbox.apache.org/repos/asf/iggy.git
commit affe6e098bbb39ee6333e61e259a81e396abab33
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Wed Oct 14 16:57:50 2026 +0200
perf(ci): avoid redundant Rust coverage builds
Rust shards and BDD jobs rebuilt tests, while coverage settings and
Python's native-library environment prevented dependency reuse.
Own Rust checks, the shared test build, and parallel shards in one
reusable workflow. Reuse it for PRs and full master coverage while
preserving affected-task selection and test failure propagation.
Instrument BDD tests during image builds and reuse Python's prebuilt
server. Collect Rust coverage only from tests and BDD, with matching
master reports and two-hour post-merge timeouts. Align master cache
warmers with the SDK jobs.
Keep compiler errors visible and avoid hashing raw coverage files.
Remove npm and Bazel caching and shorten shared setup action names.
---
.config/nextest.toml | 3 +-
.github/actions/cpp-bazel/pre-merge/action.yml | 8 +-
.github/actions/csharp-dotnet/pre-merge/action.yml | 6 +-
.github/actions/go/pre-merge/action.yml | 6 +-
.github/actions/java-gradle/post-merge/action.yml | 4 +-
.github/actions/java-gradle/pre-merge/action.yml | 8 +-
.github/actions/node-npm/pre-merge/action.yml | 7 +-
.github/actions/php/pre-merge/action.yml | 4 +-
.../actions/python-maturin/pre-merge/action.yml | 50 ++--
.github/actions/rust/post-merge/action.yml | 4 +-
.github/actions/rust/pre-merge/action.yml | 255 +++++++++++----------
.github/actions/swift/pre-merge/action.yml | 2 +-
.github/actions/utils/server-start/action.yml | 13 +-
.../{setup-cpp-with-cache => setup-cpp}/action.yml | 9 +-
.../{setup-go-with-cache => setup-go}/action.yml | 4 +-
.../action.yml | 2 +-
.../action.yml | 29 +--
.../action.yml | 86 ++++---
.../utils/validate-third-party-licenses/action.yml | 5 +-
.github/config/components.yml | 6 +-
.github/workflows/_build_python_wheels.yml | 12 +-
.github/workflows/_build_rust_artifacts.yml | 9 +-
.github/workflows/_common.yml | 1 +
.github/workflows/_detect.yml | 19 +-
.github/workflows/_publish_rust_crates.yml | 4 +-
.github/workflows/_rust.yml | 196 ++++++++++++++++
.github/workflows/_test.yml | 76 +-----
.github/workflows/_test_bdd.yml | 43 +++-
.github/workflows/_test_examples.yml | 63 +++--
.github/workflows/coverage-baseline.yml | 191 ++++++---------
.github/workflows/pr-skill-review-run.yml | 2 +
.github/workflows/pre-merge.yml | 32 +--
.github/workflows/publish.yml | 2 +-
bdd/docker-compose.cluster.yml | 3 +
bdd/docker-compose.coverage.yml | 21 +-
bdd/docker-compose.server.yml | 2 +
bdd/docker-compose.yml | 2 +
bdd/rust/Dockerfile | 40 +++-
codecov.yml | 3 +
scripts/ci/coverage-baseline-affected.sh | 2 +-
scripts/ci/storage-compat.sh | 4 +
scripts/run-bdd-tests.sh | 3 +-
42 files changed, 698 insertions(+), 543 deletions(-)
diff --git a/.config/nextest.toml b/.config/nextest.toml
index bd0f105e5..1acb8e17f 100644
--- a/.config/nextest.toml
+++ b/.config/nextest.toml
@@ -17,8 +17,7 @@
# Floor for the flags CI and scripts/ci/storage-compat.sh pass: --no-tests
# (0.9.75), --run-ignored only (0.9.76) and --ignore-default-filter (0.9.77).
-# nextest refuses to run below it, and setup-rust-with-cache replaces a
-# cache-restored binary that fails `cargo nextest show-config version`.
+# nextest refuses to run below it; setup-rust installs a pinned version above this floor.
nextest-version = { required = "0.9.77" }
[[profile.default.overrides]]
diff --git a/.github/actions/cpp-bazel/pre-merge/action.yml b/.github/actions/cpp-bazel/pre-merge/action.yml
index b21f3eb2e..eaa15c9c6 100644
--- a/.github/actions/cpp-bazel/pre-merge/action.yml
+++ b/.github/actions/cpp-bazel/pre-merge/action.yml
@@ -26,12 +26,12 @@ inputs:
runs:
using: "composite"
steps:
- - name: Setup Bazel with cache
+ - name: Setup Bazel
if: inputs.task != 'lint'
- uses: ./.github/actions/utils/setup-cpp-with-cache
+ uses: ./.github/actions/utils/setup-cpp
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
- name: Install clang-format
if: inputs.task == 'lint'
diff --git a/.github/actions/csharp-dotnet/pre-merge/action.yml b/.github/actions/csharp-dotnet/pre-merge/action.yml
index d68f930c5..d0471cd79 100644
--- a/.github/actions/csharp-dotnet/pre-merge/action.yml
+++ b/.github/actions/csharp-dotnet/pre-merge/action.yml
@@ -33,9 +33,9 @@ runs:
8.0.x
10.0.x
- - name: Setup Rust with cache
- if: inputs.task == 'test' || startsWith(inputs.task, 'e2e')
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ if: startsWith(inputs.task, 'e2e')
+ uses: ./.github/actions/utils/setup-rust
- name: Restore dependencies
run: |
diff --git a/.github/actions/go/pre-merge/action.yml b/.github/actions/go/pre-merge/action.yml
index b06f680a5..30a4a0010 100644
--- a/.github/actions/go/pre-merge/action.yml
+++ b/.github/actions/go/pre-merge/action.yml
@@ -27,7 +27,7 @@ runs:
using: "composite"
steps:
- name: Setup Go
- uses: ./.github/actions/utils/setup-go-with-cache
+ uses: ./.github/actions/utils/setup-go
- name: Format Check
shell: bash
@@ -117,9 +117,9 @@ runs:
# Build the module
go build -v ./...
- - name: Setup Rust with cache for e2e
+ - name: Setup Rust for e2e
if: inputs.task == 'e2e'
- uses: ./.github/actions/utils/setup-rust-with-cache
+ uses: ./.github/actions/utils/setup-rust
# The Go SDK speaks the VSR wire protocol, so every e2e lane runs against
# a locally built VSR server rather than a container of the legacy one.
diff --git a/.github/actions/java-gradle/post-merge/action.yml b/.github/actions/java-gradle/post-merge/action.yml
index e4e1927ea..79b7cf73e 100644
--- a/.github/actions/java-gradle/post-merge/action.yml
+++ b/.github/actions/java-gradle/post-merge/action.yml
@@ -30,8 +30,8 @@ inputs:
runs:
using: "composite"
steps:
- - name: Setup Java with cache
- uses: ./.github/actions/utils/setup-java-with-cache
+ - name: Setup Java
+ uses: ./.github/actions/utils/setup-java
- name: Build for publishing
shell: bash
diff --git a/.github/actions/java-gradle/pre-merge/action.yml b/.github/actions/java-gradle/pre-merge/action.yml
index db377bd43..a6047a619 100644
--- a/.github/actions/java-gradle/pre-merge/action.yml
+++ b/.github/actions/java-gradle/pre-merge/action.yml
@@ -26,8 +26,8 @@ inputs:
runs:
using: "composite"
steps:
- - name: Setup Java with cache
- uses: ./.github/actions/utils/setup-java-with-cache
+ - name: Setup Java
+ uses: ./.github/actions/utils/setup-java
with:
gradle-cache-disabled: "true"
@@ -76,9 +76,9 @@ runs:
# Exit with build exit code
exit $BUILD_EXIT_CODE
- - name: Setup Rust with cache
+ - name: Setup Rust
if: inputs.task == 'test'
- uses: ./.github/actions/utils/setup-rust-with-cache
+ uses: ./.github/actions/utils/setup-rust
- name: Start Iggy server
if: inputs.task == 'test'
diff --git a/.github/actions/node-npm/pre-merge/action.yml b/.github/actions/node-npm/pre-merge/action.yml
index 97771a05b..4214810f1 100644
--- a/.github/actions/node-npm/pre-merge/action.yml
+++ b/.github/actions/node-npm/pre-merge/action.yml
@@ -31,16 +31,15 @@ runs:
with:
node-version: "23"
registry-url: "https://registry.npmjs.org"
- cache: "npm"
- cache-dependency-path: foreign/node/package-lock.json
+ package-manager-cache: false
- name: Pin npm to 11 to match Dependabot lockfile format
run: npm install -g npm@11
shell: bash
- - name: Setup Rust with cache
+ - name: Setup Rust
if: inputs.task == 'e2e'
- uses: ./.github/actions/utils/setup-rust-with-cache
+ uses: ./.github/actions/utils/setup-rust
- name: Install netcat
if: inputs.task == 'e2e'
diff --git a/.github/actions/php/pre-merge/action.yml b/.github/actions/php/pre-merge/action.yml
index f69b57657..6386086a0 100644
--- a/.github/actions/php/pre-merge/action.yml
+++ b/.github/actions/php/pre-merge/action.yml
@@ -58,8 +58,8 @@ runs:
"${php_config}" --version
"${php_bin}" "${composer_bin}" --version
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
with:
shared-key: dev
save-cache: "false"
diff --git a/.github/actions/python-maturin/pre-merge/action.yml b/.github/actions/python-maturin/pre-merge/action.yml
index dcb69b146..3233c3a75 100644
--- a/.github/actions/python-maturin/pre-merge/action.yml
+++ b/.github/actions/python-maturin/pre-merge/action.yml
@@ -26,14 +26,37 @@ inputs:
runs:
using: "composite"
steps:
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
+
+ # setup-python changes PKG_CONFIG_PATH and invalidates cached native server dependencies.
+ - name: Build server Docker image for TLS tests
+ if: inputs.task == 'test'
+ run: |
+ # test_tls.py spawns the server in a container; build it from the
+ # same binaries the plain-TCP tests run against.
+ cargo build --locked --bin iggy-server --bin iggy
+ docker build \
+ -f core/server/Dockerfile \
+ --target runtime-prebuilt \
+ --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \
+ --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \
+ -t iggy-server:local .
+ shell: bash
+
+ - name: Start Iggy server
+ if: inputs.task == 'test'
+ id: iggy
+ uses: ./.github/actions/utils/server-start
+ with:
+ mode: bin
+ bin: target/debug/iggy-server
+
- name: Setup Python
uses: actions/setup-python@v6
with:
python-version: "3.11"
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
-
- name: Install cargo-llvm-cov
if: inputs.task == 'test'
uses: taiki-e/install-action@v2
@@ -147,27 +170,6 @@ runs:
uv run --no-sync maturin develop
shell: bash
- - name: Build server Docker image for TLS tests
- if: inputs.task == 'test'
- run: |
- # test_tls.py spawns the server in a container; build it from the
- # same binaries the plain-TCP tests run against.
- cargo build --locked --bin iggy-server --bin iggy
- docker build \
- -f core/server/Dockerfile \
- --target runtime-prebuilt \
- --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \
- --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \
- -t iggy-server:local .
- shell: bash
-
- - name: Start Iggy server
- if: inputs.task == 'test'
- id: iggy
- uses: ./.github/actions/utils/server-start
- with:
- cargo-bin: iggy-server
-
- name: Run Python integration tests
if: inputs.task == 'test'
run: |
diff --git a/.github/actions/rust/post-merge/action.yml b/.github/actions/rust/post-merge/action.yml
index caec167ee..13868253d 100644
--- a/.github/actions/rust/post-merge/action.yml
+++ b/.github/actions/rust/post-merge/action.yml
@@ -34,8 +34,8 @@ inputs:
runs:
using: "composite"
steps:
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
- name: Validate package
env:
diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml
index de49148fd..34d6c5d3c 100644
--- a/.github/actions/rust/pre-merge/action.yml
+++ b/.github/actions/rust/pre-merge/action.yml
@@ -20,22 +20,41 @@ description: Rust pre-merge testing and linting github iggy actions
inputs:
task:
- description: "Task to run (check, check-msrv, fmt, clippy, clippy-macos-aarch64, sort, machete, doctest, verify-publish, test-1, test-2, test-3, test-storage-compat, miri)"
+ description: "Task to run (check, check-msrv, fmt, clippy, clippy-macos-aarch64, sort, machete, doctest, verify-publish, test-build, test, test-storage-compat, miri)"
required: true
component:
description: "Component name (for context)"
required: false
default: ""
+ test-scope:
+ description: "Test selection: affected or workspace"
+ required: false
+ default: affected
+ partition:
+ description: "One-based nextest partition index for the test task"
+ required: false
+ partitions:
+ description: "Total nextest partitions for the test task"
+ required: false
+ save-cache:
+ description: "Save build dependencies for subsequent PRs"
+ required: false
+ default: "false"
+ free-disk-space-aggressive:
+ description: "Use deep disk cleanup for full workspace coverage builds"
+ required: false
+ default: "false"
runs:
using: "composite"
steps:
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
with:
# Only test jobs execute nextest. Avoid downloading it in lint, metadata,
# and cross-build jobs.
- install-nextest: ${{ startsWith(inputs.task, 'test-') }}
+ install-nextest: ${{ inputs.task == 'test' || startsWith(inputs.task, 'test-') }}
+ coverage: ${{ inputs.task == 'test-build' || inputs.task == 'test' }}
# These tasks do not compile workspace crates and need no native
# libraries. All compiling tasks keep the existing dependency setup.
install-system-dependencies: ${{ inputs.task != 'fmt' && inputs.task != 'sort' && inputs.task != 'machete' }}
@@ -48,14 +67,14 @@ runs:
# the warm `~/.cargo` restore is a free win, and its 1.95 target
# artifacts rebuild regardless since the rustc hash differs.
shared-key: ${{ inputs.task == 'miri' && 'miri' || 'dev' }}
- # fmt/sort/machete never build into target/, so the multi-GB cache
- # restore is pure overhead. Compiling legs (check/clippy/doctest/test-*)
- # keep it; a cold cache there recompiles the whole dep tree.
- read-cache: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'machete') && 'false' || 'true' }}
+ # Shards download the build artifact. Metadata-only tasks need no target cache.
+ read-cache: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'machete' || inputs.task == 'test') && 'false' || 'true' }}
+ save-cache: ${{ inputs.save-cache }}
# Light legs fit in the runner's ~89 GiB default headroom; skip the
# ~20-45s reclaim. Disk-heavy legs (coverage build + testcontainers
- # images on test-*, cross-builds, miri, verify-publish) keep it.
+ # images on test jobs, cross-builds, miri, verify-publish) keep it.
free-disk-space: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'check-msrv' || inputs.task == 'machete' || inputs.task == 'doctest') && 'false' || 'true' }}
+ free-disk-space-aggressive: ${{ inputs.free-disk-space-aggressive }}
- name: Install cargo-sort
if: inputs.task == 'sort'
@@ -84,7 +103,7 @@ runs:
# Cargo-Rail release it is qualified with (0.30.1 for v10.1.1).
- name: Plan affected Rust work
id: rail
- if: startsWith(inputs.task, 'test-') && inputs.task != 'test-storage-compat'
+ if: inputs.task == 'test-build' && inputs.test-scope == 'affected'
continue-on-error: true
uses: loadingalias/cargo-rail-action@409962f05d90a78aae995ef6b1c8466010c0a460 # v10.1.1
@@ -149,45 +168,22 @@ runs:
shell: bash
- name: Install dependencies for Rust tests
- if: startsWith(inputs.task, 'test-') && runner.os == 'Linux'
+ if: (inputs.task == 'test' || inputs.task == 'test-storage-compat') && runner.os == 'Linux'
run: |
./scripts/ci/apt-install.sh musl-tools gnome-keyring keyutils dbus-x11 libsecret-tools
rm -f "$HOME"/.local/share/keyrings/*
shell: bash
- - name: Install cargo-llvm-cov
- if: startsWith(inputs.task, 'test-') && inputs.task != 'test-storage-compat'
- uses: taiki-e/install-action@v2
- with:
- tool: cargo-llvm-cov
-
- - name: Build and test with coverage
- # test-storage-compat drives its own nextest invocation from
- # scripts/ci/storage-compat.sh. Without this exclusion it would also run
- # here without partition arguments, i.e. a fourth *unpartitioned* copy
- # of the whole suite on top of test-1/2/3.
- if: startsWith(inputs.task, 'test-') && inputs.task != 'test-storage-compat'
+ - name: Build coverage test archive
+ if: inputs.task == 'test-build'
env:
PLAN_FILE: ${{ steps.rail.outputs.plan-file }}
PLAN_STATUS: ${{ steps.rail.outcome }}
- TASK: ${{ inputs.task }}
+ TEST_SELECTION: ${{ inputs.test-scope }}
run: |
set -euo pipefail
-
- # Parse partition index from task name (test-1 -> hash:1/3, test-2 -> hash:2/3, ...).
- # TEST_PARTITIONS must match the number of test-N tasks in
- # .github/config/components.yml. Cluster bootstrap makes each test
- # CPU-heavy, so partitions stay small.
- # The regex is numeric-only on purpose: non-numeric test-* tasks
- # (test-storage-compat) opt out of partitioning rather than producing
- # an out-of-range hash:N/3.
- TEST_PARTITIONS=3
- PARTITION_ARGS=()
- if [[ "$TASK" =~ ^test-([0-9]+)$ ]]; then
- PARTITION_INDEX="${BASH_REMATCH[1]}"
- PARTITION_ARGS=(--partition "hash:${PARTITION_INDEX}/${TEST_PARTITIONS}")
- echo "::notice::Running test partition ${PARTITION_INDEX}/${TEST_PARTITIONS}"
- fi
+ ARTIFACT_DIR="$RUNNER_TEMP/rust-test-build"
+ mkdir -p "$ARTIFACT_DIR"
# Read only validated selectors. Argument and package selectors are
# NUL-delimited, hence `mapfile -d ''`. Any planner or selector failure
@@ -195,12 +191,18 @@ runs:
# target-args emits `--test` pairs only when every selected target of
# every selected package is an integration test, so it never drops a
# package or its unit tests from a mixed selection.
- SELECTOR_DIR="${RUNNER_TEMP:?RUNNER_TEMP is required}/cargo-rail-selectors-${TASK}"
+ SELECTOR_DIR="${RUNNER_TEMP:?RUNNER_TEMP is required}/cargo-rail-selectors"
mkdir -p "$SELECTOR_DIR"
TEST_CARGO_ARGS=()
TEST_TARGET_ARGS=()
TEST_PACKAGES=()
- if [[ "$PLAN_STATUS" == "success" ]] &&
+ if [[ "$TEST_SELECTION" == "workspace" ]]; then
+ TEST_SCOPE=workspace
+ RUNTIME_SCOPE=workspace
+ elif [[ "$TEST_SELECTION" != "affected" ]]; then
+ echo "::error::Invalid test scope: $TEST_SELECTION"
+ exit 1
+ elif [[ "$PLAN_STATUS" == "success" ]] &&
TEST_SCOPE=$(cargo-rail-action plan cargo-scope "$PLAN_FILE" cargo.test) &&
RUNTIME_SCOPE=$(cargo-rail-action plan cargo-scope "$PLAN_FILE" runtime-artifacts) &&
cargo-rail-action plan cargo-args "$PLAN_FILE" cargo.test > "$SELECTOR_DIR/test-args" &&
@@ -216,61 +218,44 @@ runs:
echo "::warning::Cargo-Rail plan unavailable or invalid; running the full workspace"
fi
+ printf '%s\n' "$TEST_SCOPE" > "$ARTIFACT_DIR/scope"
+ git rev-parse HEAD > "$ARTIFACT_DIR/revision"
if [[ "$TEST_SCOPE" == "skipped" ]]; then
echo "::notice::Cargo-Rail reports no affected Rust tests"
exit 0
fi
- # Runtime prerequisites (spawned binaries, dlopened connector plugins)
- # are built alongside the tested packages. A package in both lists is
- # built once.
+ # Only spawned binaries and dlopened plugins need a separate dev build.
+ # nextest archive compiles the selected tests with their dev-dependency features.
BUILD_ARGS=()
- if [[ "$TEST_SCOPE" == "packages" && "$RUNTIME_SCOPE" != "workspace" ]]; then
- mapfile -d '' -t BUILD_PACKAGES < <(sort -zu "$SELECTOR_DIR/test-packages" "$SELECTOR_DIR/runtime-packages")
- for package in "${BUILD_PACKAGES[@]}"; do
+ if [[ "$RUNTIME_SCOPE" == "packages" ]]; then
+ mapfile -d '' -t RUNTIME_PACKAGES < "$SELECTOR_DIR/runtime-packages"
+ for package in "${RUNTIME_PACKAGES[@]}"; do
BUILD_ARGS+=(-p "$package")
done
- BUILD_SUMMARY="${#BUILD_PACKAGES[@]} packages"
- else
- BUILD_SUMMARY="full workspace"
fi
- if [[ "$TEST_SCOPE" == "packages" ]]; then
- TEST_SUMMARY="${#TEST_PACKAGES[@]} packages"
- else
- TEST_SUMMARY="full workspace"
- fi
- echo "::notice::Cargo-Rail: testing ${TEST_SUMMARY}, building ${BUILD_SUMMARY}"
-
- source <(cargo llvm-cov show-env --export-prefix)
-
- # Doris 4.0.3's start_be.sh hard-`exit 1`s unless vm.max_map_count >= 2000000.
- # This kernel param can only be set on the host (no container can raise it),
- # so we raise it here. Everything else about booting Doris — image, heap/mem
- # caps, port mappings, BE-alive wait — lives in the testcontainers fixture
- # at core/integration/tests/connectors/fixtures/doris/container.rs, so
- # `cargo test` and CI follow exactly the same path.
- if [[ "$RUNNER_OS" == "Linux" ]]; then
- sudo sysctl -w vm.max_map_count=2000000 || true
+ : > "$ARTIFACT_DIR/build.jsonl"
+ : > "$ARTIFACT_DIR/runtime-files"
+ if [[ "$RUNTIME_SCOPE" != "skipped" ]]; then
+ cargo build --locked "${BUILD_ARGS[@]}" --message-format=json-render-diagnostics > "$ARTIFACT_DIR/build.jsonl"
+ jq -r 'select(.reason == "compiler-artifact") |
+ (select(.target.kind | index("bin")) | .executable // empty),
+ (select(.target.crate_types | index("cdylib")) | .filenames[] | select(endswith(".so")))' \
+ "$ARTIFACT_DIR/build.jsonl" | sort -u | \
+ while IFS= read -r artifact; do
+ realpath --relative-to="$GITHUB_WORKSPACE" "$artifact"
+ done > "$ARTIFACT_DIR/runtime-files"
fi
-
- bins_start=$(date +%s)
- cargo build --locked "${BUILD_ARGS[@]}"
- bins_end=$(date +%s)
- bins_duration=$((bins_end - bins_start))
- echo "::notice::Binaries and libraries built in ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))"
-
- compile_start=$(date +%s)
- cargo test --locked --no-run "${TEST_CARGO_ARGS[@]}" "${TEST_TARGET_ARGS[@]}"
- compile_end=$(date +%s)
- compile_duration=$((compile_end - compile_start))
- echo "::notice::Tests compiled in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))"
+ tar --zstd -cf "$ARTIFACT_DIR/runtime.tar.zst" -T "$ARTIFACT_DIR/runtime-files"
+ rm "$ARTIFACT_DIR/build.jsonl" "$ARTIFACT_DIR/runtime-files"
# api_handler_tests, version_firewall_tests, and server_e2e_tests need gitignored
- # wire fixtures. Generate when iggy-gateway-kafka is in the Cargo-Rail test scope, or (on
- # a full-workspace run) when gateways/** changed vs the PR base — avoid building
- # kafka-message-gen / kafka-protocol when the gateway was not touched.
+ # wire fixtures. Master always generates them. On PRs, use the selected
+ # packages or the gateway diff to avoid building fixture tools unnecessarily.
NEEDS_KAFKA_FIXTURES=false
- if [[ "$TEST_SCOPE" == "packages" ]]; then
+ if [[ "$TEST_SELECTION" == "workspace" ]]; then
+ NEEDS_KAFKA_FIXTURES=true
+ elif [[ "$TEST_SCOPE" == "packages" ]]; then
for package in "${TEST_PACKAGES[@]}"; do
if [[ "$package" == "iggy-gateway-kafka" ]]; then
NEEDS_KAFKA_FIXTURES=true
@@ -313,9 +298,44 @@ runs:
echo "::warning::git diff against the PR base failed (base: ${BASE_SHA:-none, HEAD is not a merge commit}; ${DIFF_OUTPUT:-}); generating Kafka fixtures unconditionally as a fail-safe"
fi
fi
+ printf '%s\n' "$NEEDS_KAFKA_FIXTURES" > "$ARTIFACT_DIR/kafka-fixtures"
if [[ "$NEEDS_KAFKA_FIXTURES" == true ]]; then
./gateways/kafka/scripts/ci-wire-fixtures.sh generate
- trap './gateways/kafka/scripts/ci-wire-fixtures.sh cleanup' EXIT
+ tar --zstd -cf "$ARTIFACT_DIR/kafka-fixtures.tar.zst" gateways/kafka/tools/kafka-tool/kafka_messages
+ fi
+
+ cargo nextest archive --locked --profile ci \
+ "${TEST_CARGO_ARGS[@]}" "${TEST_TARGET_ARGS[@]}" \
+ --archive-file "$ARTIFACT_DIR/nextest.tar.zst"
+ shell: bash
+
+ - name: Run coverage test partition
+ if: inputs.task == 'test'
+ env:
+ TEST_PARTITION: ${{ inputs.partition }}
+ TEST_PARTITIONS: ${{ inputs.partitions }}
+ run: |
+ set -euo pipefail
+ if [[ ! "$TEST_PARTITION" =~ ^[1-9][0-9]*$ || ! "$TEST_PARTITIONS" =~ ^[1-9][0-9]*$ ]] ||
+ (( TEST_PARTITION > TEST_PARTITIONS )); then
+ echo "::error::Invalid test partition: $TEST_PARTITION/$TEST_PARTITIONS"
+ exit 1
+ fi
+ ARTIFACT_DIR="$RUNNER_TEMP/rust-test-build"
+ artifact_revision=$(cat "$ARTIFACT_DIR/revision")
+ checkout_revision=$(git rev-parse HEAD)
+ if [[ "$artifact_revision" != "$checkout_revision" ]]; then
+ echo "::error::Coverage archive revision $artifact_revision does not match checkout $checkout_revision"
+ exit 1
+ fi
+ TEST_SCOPE=$(cat "$ARTIFACT_DIR/scope")
+ if [[ "$TEST_SCOPE" == "skipped" ]]; then
+ echo "::notice::Cargo-Rail reports no affected Rust tests"
+ exit 0
+ fi
+ tar --zstd -xf "$ARTIFACT_DIR/runtime.tar.zst"
+ if [[ "$(cat "$ARTIFACT_DIR/kafka-fixtures")" == true ]]; then
+ tar --zstd -xf "$ARTIFACT_DIR/kafka-fixtures.tar.zst"
# Fixtures are expected to exist from here on; a missing one now means
# generation silently failed, not a legitimate local-dev skip.
export KAFKA_FIXTURES_REQUIRED=1
@@ -349,52 +369,23 @@ runs:
# Previously this ran before `cargo build`, leaving a 7+ minute idle
# window that triggered org.freedesktop.Secret.Error.IsLocked ~10% of runs.
if [[ "$RUNNER_OS" == "Linux" ]]; then
- eval $(dbus-launch --sh-syntax)
+ eval "$(dbus-launch --sh-syntax)"
export DBUS_SESSION_BUS_ADDRESS
- eval $(echo -n "test" | gnome-keyring-daemon --unlock --components=secrets)
+ eval "$(echo -n "test" | gnome-keyring-daemon --unlock --components=secrets)"
echo -n "warmup" | secret-tool store --label="ci-warmup" ci-test warmup
fi
- test_start=$(date +%s)
- if command -v cargo-nextest &> /dev/null; then
- if [[ "$TEST_SCOPE" == "packages" ]]; then
- # Every test target in the `bdd` crate is `required-features =
- # ["bdd"]`, so the `cargo test --no-run` above builds no binaries
- # for it. A PR touching only that crate then produces a valid
- # package scope that matches zero tests, and nextest's default
- # `--no-tests=fail` turns that into exit 4 on a green PR.
- cargo nextest run --locked --no-fail-fast --no-tests=warn --profile ci \
- "${PARTITION_ARGS[@]}" "${TEST_CARGO_ARGS[@]}" "${TEST_TARGET_ARGS[@]}"
- else
- cargo nextest run --locked --no-fail-fast --profile ci "${PARTITION_ARGS[@]}"
- fi
- else
- if [[ ${#PARTITION_ARGS[@]} -gt 0 ]]; then
- echo "::error::cargo-nextest not found, falling back to cargo test without partitioning (all tests will run on every partition)"
- fi
- cargo test --locked --no-fail-fast "${TEST_CARGO_ARGS[@]}" "${TEST_TARGET_ARGS[@]}"
+ # Doris requires this host setting before its testcontainers fixture starts.
+ sudo sysctl -w vm.max_map_count=2000000 || true
+ NO_TESTS=fail
+ if [[ "$TEST_SCOPE" == "packages" ]]; then
+ # BDD targets require features, so a BDD-only scope can contain no runnable tests.
+ NO_TESTS=warn
fi
- test_end=$(date +%s)
- test_duration=$((test_end - test_start))
- echo "::notice::Tests executed in ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))"
-
- build_duration=$((bins_duration + compile_duration))
- total_duration=$((build_duration + test_duration))
- echo ""
- echo "========================================="
- echo "Cargo-Rail scope (test): ${TEST_SUMMARY}"
- echo "Cargo-Rail scope (build): ${BUILD_SUMMARY}"
- echo "All targets build: ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))"
- echo "Tests compile: ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))"
- echo "Tests execute: ${test_duration}s ($(date -ud @${test_duration} +'%M:%S'))"
- echo "-----------------------------------------"
- echo "Total build: ${build_duration}s ($(date -ud @${build_duration} +'%M:%S'))"
- echo "Total time: ${total_duration}s ($(date -ud @${total_duration} +'%M:%S'))"
- echo "========================================="
-
- cargo llvm-cov report --codecov --output-path codecov.json
- echo "Coverage report generated: codecov.json"
- ls -la codecov.json
+ cargo nextest run --archive-file "$ARTIFACT_DIR/nextest.tar.zst" \
+ --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --workspace-remap "$GITHUB_WORKSPACE" \
+ --no-fail-fast --no-tests="$NO_TESTS" --profile ci \
+ --partition "hash:${TEST_PARTITION}/${TEST_PARTITIONS}"
shell: bash
# On-disk format backwards compatibility. The script builds an iggy-server
@@ -478,7 +469,19 @@ runs:
run: |
target/debug/iggy-server --fresh > server-boot.log 2>&1 &
SERVER_PID=$!
- trap 'kill "$SERVER_PID" 2>/dev/null || true' EXIT
+ stop_server() {
+ local stop_timeout=30
+ kill "$SERVER_PID" 2>/dev/null || true
+ for ((elapsed = 0; elapsed < stop_timeout; elapsed++)); do
+ if ! kill -0 "$SERVER_PID" 2>/dev/null; then
+ break
+ fi
+ sleep 1
+ done
+ kill -KILL "$SERVER_PID" 2>/dev/null || true
+ wait "$SERVER_PID" || true
+ }
+ trap stop_server EXIT
for _ in $(seq 1 30); do
if ! kill -0 "$SERVER_PID" 2>/dev/null; then
diff --git a/.github/actions/swift/pre-merge/action.yml b/.github/actions/swift/pre-merge/action.yml
index f21a11903..8623a6a53 100644
--- a/.github/actions/swift/pre-merge/action.yml
+++ b/.github/actions/swift/pre-merge/action.yml
@@ -80,7 +80,7 @@ runs:
# the lane skips the disk cleanup, the apt install, and the cache restore.
- name: Setup Rust for the golden vectors
if: inputs.task == 'golden'
- uses: ./.github/actions/utils/setup-rust-with-cache
+ uses: ./.github/actions/utils/setup-rust
with:
free-disk-space: "false"
read-cache: "false"
diff --git a/.github/actions/utils/server-start/action.yml b/.github/actions/utils/server-start/action.yml
index 664582f7e..6b76e9cc5 100644
--- a/.github/actions/utils/server-start/action.yml
+++ b/.github/actions/utils/server-start/action.yml
@@ -112,14 +112,13 @@ runs:
else
OUT="target/debug/$NAME"
fi
- if [[ ! -x "$OUT" ]]; then
- echo "Building $NAME with cargo ($PROFILE)…"
- CARGO_ARGS=(build --locked --bin "$NAME")
- if [[ "$PROFILE" == "release" ]]; then
- CARGO_ARGS+=(--release)
- fi
- cargo "${CARGO_ARGS[@]}"
+ # An existing executable may belong to a different checkout or feature set.
+ echo "Building $NAME with cargo ($PROFILE)..."
+ CARGO_ARGS=(build --locked --bin "$NAME")
+ if [[ "$PROFILE" == "release" ]]; then
+ CARGO_ARGS+=(--release)
fi
+ cargo "${CARGO_ARGS[@]}"
BIN_PATH="$OUT"
else
BIN_PATH="${{ inputs.bin }}"
diff --git a/.github/actions/utils/setup-cpp-with-cache/action.yml b/.github/actions/utils/setup-cpp/action.yml
similarity index 82%
rename from .github/actions/utils/setup-cpp-with-cache/action.yml
rename to .github/actions/utils/setup-cpp/action.yml
index 1ffb584bf..9f3a43bfd 100644
--- a/.github/actions/utils/setup-cpp-with-cache/action.yml
+++ b/.github/actions/utils/setup-cpp/action.yml
@@ -15,18 +15,15 @@
# specific language governing permissions and limitations
# under the License.
-name: setup-cpp-with-cache
-description: Setup Bazel toolchain and caching (bazelisk + repository + disk cache)
+name: setup-cpp
+description: Setup Bazel toolchain
runs:
using: "composite"
steps:
- - name: Setup Bazel with cache
+ - name: Setup Bazel
uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # v0.19.0
with:
- bazelisk-cache: true
- disk-cache: true
- repository-cache: true
module-root: foreign/cpp
- name: Verify Bazel installation
diff --git a/.github/actions/utils/setup-go-with-cache/action.yml b/.github/actions/utils/setup-go/action.yml
similarity index 98%
rename from .github/actions/utils/setup-go-with-cache/action.yml
rename to .github/actions/utils/setup-go/action.yml
index 10f9c2a7e..ae896bbf9 100644
--- a/.github/actions/utils/setup-go-with-cache/action.yml
+++ b/.github/actions/utils/setup-go/action.yml
@@ -16,10 +16,10 @@
# under the License.
# NOTE: Go builds are fast, so caching is disabled by default.
-# The read-cache/save-cache inputs exist for parity with setup-rust-with-cache
+# The read-cache/save-cache inputs exist for parity with setup-rust
# but both default to "false". Enable only if Go build times become a bottleneck.
-name: setup-go-with-cache
+name: setup-go
description: Setup Go toolchain and optional caching (go mod cache + golangci-lint cache)
inputs:
go-version:
diff --git a/.github/actions/utils/setup-java-with-cache/action.yml b/.github/actions/utils/setup-java/action.yml
similarity index 98%
rename from .github/actions/utils/setup-java-with-cache/action.yml
rename to .github/actions/utils/setup-java/action.yml
index 2a757b24e..e9f60cd28 100644
--- a/.github/actions/utils/setup-java-with-cache/action.yml
+++ b/.github/actions/utils/setup-java/action.yml
@@ -18,7 +18,7 @@
# NOTE: this action sets up the Java toolchain with optional Gradle caching.
# Gradle itself is provided via the wrapper (gradlew) in each project.
-name: setup-java-with-cache
+name: setup-java
description: Setup Java toolchain with optional Gradle caching (uses gradlew wrapper)
inputs:
java-version:
diff --git a/.github/actions/utils/setup-node-with-cache/action.yml b/.github/actions/utils/setup-node/action.yml
similarity index 68%
rename from .github/actions/utils/setup-node-with-cache/action.yml
rename to .github/actions/utils/setup-node/action.yml
index a46b199e4..4c4767694 100644
--- a/.github/actions/utils/setup-node-with-cache/action.yml
+++ b/.github/actions/utils/setup-node/action.yml
@@ -15,24 +15,13 @@
# specific language governing permissions and limitations
# under the License.
-# NOTE: this action sets up the Node.js toolchain + npm cache for dependencies,
-# it is a convenience wrapper, so that we can use it in all workflows.
-
-name: setup-node-with-cache
-description: Setup Node.js toolchain and comprehensive caching (npm cache for dependencies)
+name: setup-node
+description: Setup Node.js toolchain and install dependencies
inputs:
node-version:
description: "Node.js version to use"
required: false
default: "23"
- enabled:
- description: "Whether to enable caching"
- required: false
- default: "true"
- cache-dependency-path:
- description: "Path to lock file for npm caching"
- required: false
- default: "package-lock.json"
workspace:
description: "Workspace directory"
required: false
@@ -44,25 +33,13 @@ runs:
uses: actions/setup-node@v6
with:
node-version: ${{ inputs.node-version }}
- cache: "npm"
- cache-dependency-path: ${{ inputs.cache-dependency-path }}
+ package-manager-cache: false
- name: Pin npm to 11 to match Dependabot lockfile format
run: npm install -g npm@11
shell: bash
- - name: Setup npm cache
- if: inputs.enabled == 'true'
- uses: actions/cache@v5
- with:
- path: ~/.npm
- key: npm-${{ runner.os }}-${{ hashFiles(inputs.cache-dependency-path) }}
- restore-keys: |
- npm-${{ runner.os }}-
-
-
- name: Install dependencies
- if: inputs.enabled == 'true'
run: |
echo "📦 Installing Node.js dependencies..."
if [ "${{ inputs.workspace }}" != "." ]; then
diff --git a/.github/actions/utils/setup-rust-with-cache/action.yml b/.github/actions/utils/setup-rust/action.yml
similarity index 74%
rename from .github/actions/utils/setup-rust-with-cache/action.yml
rename to .github/actions/utils/setup-rust/action.yml
index 023483300..c4f0a2026 100644
--- a/.github/actions/utils/setup-rust-with-cache/action.yml
+++ b/.github/actions/utils/setup-rust/action.yml
@@ -15,10 +15,14 @@
# specific language governing permissions and limitations
# under the License.
-name: setup-rust-with-cache
+name: setup-rust
description: Setup Rust toolchain with Swatinem/rust-cache
inputs:
+ coverage:
+ description: "Instrument workspace crates for coverage before compiling"
+ required: false
+ default: "false"
install-nextest:
description: "Whether to install cargo-nextest"
required: false
@@ -88,15 +92,35 @@ runs:
rustup-init -y --no-modify-path --default-toolchain none
fi
rustup show
+ # rust-cache hashes every installed toolchain, including unused runner defaults.
+ active_toolchain=$(rustup show active-toolchain)
+ active_toolchain=${active_toolchain%% *}
+ installed_toolchains=$(rustup toolchain list --quiet)
+ while read -r toolchain; do
+ if [[ "$toolchain" != "$active_toolchain" ]]; then
+ rustup toolchain uninstall "$toolchain"
+ fi
+ done <<< "$installed_toolchains"
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
shell: bash
+ - name: Configure Cargo for CI
+ run: |
+ echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV"
+ echo "CARGO_PROFILE_DEV_DEBUG=0" >> "$GITHUB_ENV"
+ echo "CARGO_PROFILE_TEST_DEBUG=0" >> "$GITHUB_ENV"
+ echo "CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse" >> "$GITHUB_ENV"
+ shell: bash
+
+ # llvm-cov instruments workspace crates, which rust-cache removes before saving.
+ # External dependencies can share the cache with check, lint and SDK server builds.
- name: Setup Rust cache
id: rust-cache
if: inputs.read-cache == 'true'
uses: Swatinem/rust-cache@v2
with:
shared-key: ${{ inputs.shared-key }}
+ cache-workspace-crates: "false"
save-if: ${{ inputs.save-cache == 'true' }}
- name: Cache status
@@ -123,42 +147,32 @@ runs:
shell: bash
- name: Install cargo-nextest
- if: runner.os == 'Linux' && inputs.install-nextest == 'true'
- run: |
- # A binary restored from the cargo cache can predate the floor in
- # .config/nextest.toml. `show-config version` exits non-zero below it,
- # and on a nextest too old to know the subcommand at all, so both fall
- # through to a fresh install instead of aborting the lane later.
- if command -v cargo-nextest &> /dev/null && cargo nextest show-config version; then
- echo "cargo-nextest already installed"
- exit 0
- fi
-
- ARCH="$(uname -m)"
- case "$ARCH" in
- x86_64) NEXTEST_PLATFORM="linux" ;;
- aarch64) NEXTEST_PLATFORM="linux-arm" ;;
- *)
- echo "Unsupported architecture: $ARCH, skipping nextest"
- exit 0
- ;;
- esac
+ if: inputs.install-nextest == 'true'
+ uses: taiki-e/install-action@v2
+ with:
+ tool: cargo-nextest@0.9.148
- CARGO_BIN="${CARGO_HOME:-$HOME/.cargo}/bin"
- tarball="$(mktemp)"
- # Staged to a file because curl --retry cannot rewind a pipe.
- curl -LsSf --retry 5 --retry-all-errors -o "$tarball" "https://get.nexte.st/latest/${NEXTEST_PLATFORM}"
- tar xzf "$tarball" -C "$CARGO_BIN"
- rm -f "$tarball"
- cargo nextest --version
- shell: bash
- continue-on-error: true
+ - name: Install cargo-llvm-cov
+ if: inputs.coverage == 'true'
+ uses: taiki-e/install-action@v2
+ with:
+ tool: cargo-llvm-cov@0.9.1
- - name: Configure Cargo for CI
- if: inputs.read-cache == 'true'
+ - name: Enable coverage for subsequent builds and tests
+ if: inputs.coverage == 'true'
run: |
- echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV"
- echo "CARGO_PROFILE_DEV_DEBUG=0" >> "$GITHUB_ENV"
- echo "CARGO_PROFILE_TEST_DEBUG=0" >> "$GITHUB_ENV"
- echo "CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse" >> "$GITHUB_ENV"
+ set -euo pipefail
+ rustup component add llvm-tools-preview
+ coverage_env=$(cargo llvm-cov show-env --sh)
+ eval "$coverage_env"
+ if [[ "${{ inputs.read-cache }}" == true ]]; then
+ cargo llvm-cov clean --workspace
+ fi
+ while IFS= read -r line; do
+ if [[ "$line" =~ ^export\ ([A-Za-z_][A-Za-z0-9_]*)= ]]; then
+ name=${BASH_REMATCH[1]}
+ delimiter="COVERAGE_ENV_${RANDOM}_${RANDOM}"
+ printf '%s<<%s\n%s\n%s\n' "$name" "$delimiter" "${!name}" "$delimiter" >> "$GITHUB_ENV"
+ fi
+ done <<< "$coverage_env"
shell: bash
diff --git a/.github/actions/utils/validate-third-party-licenses/action.yml b/.github/actions/utils/validate-third-party-licenses/action.yml
index 772da695d..ca853e53b 100644
--- a/.github/actions/utils/validate-third-party-licenses/action.yml
+++ b/.github/actions/utils/validate-third-party-licenses/action.yml
@@ -37,9 +37,10 @@ runs:
uses: actions/setup-node@v6
with:
node-version: ${{ inputs.node-version }}
+ package-manager-cache: false
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
with:
read-cache: "false"
diff --git a/.github/config/components.yml b/.github/config/components.yml
index b8c5dd036..4714969fe 100644
--- a/.github/config/components.yml
+++ b/.github/config/components.yml
@@ -144,7 +144,7 @@ components:
# existing deployments, which no other check covers. Matching is literal
# glob with no crate-graph expansion, so every crate has to be spelled out:
# `core/server/**` does not cover `core/server_common/**`.
- # `breaking:storage` on the PR skips it (see .github/workflows/_test.yml).
+ # `breaking:storage` on the PR skips it (see .github/workflows/_rust.yml).
rust-storage-compat:
depends_on:
- "rust-workspace"
@@ -212,9 +212,7 @@ components:
- "sort"
- "doctest"
- "machete"
- - "test-1"
- - "test-2"
- - "test-3"
+ - "test"
- "build-aarch64-gnu"
- "build-aarch64-musl"
- "build-macos-aarch64"
diff --git a/.github/workflows/_build_python_wheels.yml b/.github/workflows/_build_python_wheels.yml
index 1942a1ba1..af8e43c57 100644
--- a/.github/workflows/_build_python_wheels.yml
+++ b/.github/workflows/_build_python_wheels.yml
@@ -91,8 +91,8 @@ jobs:
id: interpreters
run: ./scripts/ci/wheel-interpreters.sh >> "$GITHUB_OUTPUT"
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
- name: Generate third-party license manifest
run: |
@@ -166,8 +166,8 @@ jobs:
with:
python-version: ${{ steps.interpreters.outputs.python_versions }}
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
- name: Generate third-party license manifest
run: |
@@ -240,8 +240,8 @@ jobs:
python-version: ${{ matrix.python-version }}
architecture: x64
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
with:
shared-key: python-wheels-windows
diff --git a/.github/workflows/_build_rust_artifacts.yml b/.github/workflows/_build_rust_artifacts.yml
index 5d8715f63..df77bba9c 100644
--- a/.github/workflows/_build_rust_artifacts.yml
+++ b/.github/workflows/_build_rust_artifacts.yml
@@ -102,8 +102,8 @@ jobs:
if: matrix.libc == 'musl'
run: ./scripts/ci/apt-install.sh musl-tools
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
- name: Add Rust target
run: rustup target add ${{ matrix.target }}
@@ -112,6 +112,7 @@ jobs:
uses: actions/setup-node@v7
with:
node-version: "22"
+ package-manager-cache: false
- name: Build Web UI static files
run: npm --prefix web ci && npm --prefix web run build:static
@@ -213,8 +214,8 @@ jobs:
/tmp/copy-latest-from-master.sh save .github scripts
/tmp/copy-latest-from-master.sh apply
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
- name: Add Rust target
run: rustup target add ${{ matrix.target }}
diff --git a/.github/workflows/_common.yml b/.github/workflows/_common.yml
index 56459044a..e4e247cdc 100644
--- a/.github/workflows/_common.yml
+++ b/.github/workflows/_common.yml
@@ -151,6 +151,7 @@ jobs:
uses: actions/setup-node@v7
with:
node-version: "23"
+ package-manager-cache: false
- name: Install markdownlint-cli
run: npm install -g markdownlint-cli
diff --git a/.github/workflows/_detect.yml b/.github/workflows/_detect.yml
index 38902b859..8cd417c1c 100644
--- a/.github/workflows/_detect.yml
+++ b/.github/workflows/_detect.yml
@@ -23,9 +23,9 @@ on:
third_party_licenses:
description: "Whether third-party license inputs changed"
value: ${{ jobs.detect.outputs.third_party_licenses }}
- rust_matrix:
- description: "Matrix for Rust components"
- value: ${{ jobs.detect.outputs.rust_matrix }}
+ rust_plan:
+ description: "Selected Rust checks and tests; empty when Rust is unaffected"
+ value: ${{ jobs.detect.outputs.rust_plan }}
python_matrix:
description: "Matrix for Python SDK"
value: ${{ jobs.detect.outputs.python_matrix }}
@@ -69,7 +69,7 @@ jobs:
timeout-minutes: 10
outputs:
third_party_licenses: ${{ github.event_name != 'pull_request' || steps.changed.outputs.third_party_licenses == 'true' }}
- rust_matrix: ${{ steps.mk.outputs.rust_matrix }}
+ rust_plan: ${{ steps.mk.outputs.rust_plan }}
python_matrix: ${{ steps.mk.outputs.python_matrix }}
php_matrix: ${{ steps.mk.outputs.php_matrix }}
node_matrix: ${{ steps.mk.outputs.node_matrix }}
@@ -111,7 +111,7 @@ jobs:
- 'scripts/ci/render-node-licenses.mjs'
- 'scripts/ci/apt-install.sh'
- '.github/actions/utils/validate-third-party-licenses/**'
- - '.github/actions/utils/setup-rust-with-cache/**'
+ - '.github/actions/utils/setup-rust/**'
- '.github/actions/utils/free-disk-space/**'
- '.github/workflows/_detect.yml'
- '.github/workflows/_common.yml'
@@ -290,7 +290,7 @@ jobs:
const entries = cfg.tasks.map(task => ({ component: name, task }));
taskedComponents.push({ name, tasks: cfg.tasks, trigger });
- if (name === 'rust') groups.rust.push(...entries);
+ if (name === 'rust' || name.startsWith('rust-')) groups.rust.push(...entries);
else if (name === 'sdk-python') groups.python.push(...entries);
else if (name === 'sdk-php') groups.php.push(...entries);
else if (name === 'sdk-node') groups.node.push(...entries);
@@ -354,7 +354,7 @@ jobs:
for (const [name, cfg] of Object.entries(components)) {
if (!cfg.tasks || cfg.tasks.length === 0) continue;
const entries = cfg.tasks.map(task => ({ component: name, task }));
- if (name === 'rust') groups.rust.push(...entries);
+ if (name === 'rust' || name.startsWith('rust-')) groups.rust.push(...entries);
else if (name === 'sdk-python') groups.python.push(...entries);
else if (name === 'sdk-php') groups.php.push(...entries);
else if (name === 'sdk-node') groups.node.push(...entries);
@@ -426,7 +426,10 @@ jobs:
require('fs').appendFileSync(process.env.GITHUB_OUTPUT, `${output}\n`);
};
- setOutput('rust_matrix', JSON.stringify(matrix(groups.rust)));
+ setOutput('rust_plan', groups.rust.length ? JSON.stringify({
+ checks: groups.rust.filter(entry => entry.task !== 'test'),
+ tests: groups.rust.some(entry => entry.task === 'test'),
+ }) : '');
setOutput('python_matrix', JSON.stringify(matrix(groups.python)));
setOutput('php_matrix', JSON.stringify(matrix(groups.php)));
setOutput('node_matrix', JSON.stringify(matrix(groups.node)));
diff --git a/.github/workflows/_publish_rust_crates.yml b/.github/workflows/_publish_rust_crates.yml
index ae2a8d739..2f268231d 100644
--- a/.github/workflows/_publish_rust_crates.yml
+++ b/.github/workflows/_publish_rust_crates.yml
@@ -112,8 +112,8 @@ jobs:
/tmp/copy-latest-from-master.sh apply
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
# Normalize the inbound commit to a canonical full SHA. Caller (publish.yml)
# already does this; a direct workflow_call caller must pass an explicit
diff --git a/.github/workflows/_rust.yml b/.github/workflows/_rust.yml
new file mode 100644
index 000000000..7d5458b72
--- /dev/null
+++ b/.github/workflows/_rust.yml
@@ -0,0 +1,196 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: _rust
+on:
+ workflow_call:
+ inputs:
+ plan:
+ description: "JSON with checks (component/task entries) and tests (boolean)"
+ type: string
+ required: true
+ test-scope:
+ description: "Test selection: affected or workspace"
+ type: string
+ default: affected
+ timeout-minutes:
+ type: number
+ default: 60
+ save-cache:
+ description: "Save dependencies from the test build for subsequent PRs"
+ type: boolean
+ default: false
+ build-metadata:
+ description: "Embed CI build metadata in the server binaries"
+ type: boolean
+ default: false
+ secrets:
+ CODECOV_TOKEN:
+ required: false
+
+permissions:
+ contents: read
+ issues: read # live breaking:storage label lookup
+
+env:
+ IGGY_CI_BUILD: ${{ inputs.build-metadata }}
+
+jobs:
+ checks:
+ name: ${{ matrix.component }}/${{ matrix.task }}
+ if: ${{ fromJSON(inputs.plan).checks[0] != null }}
+ strategy:
+ fail-fast: true
+ matrix:
+ include: ${{ fromJSON(inputs.plan).checks }}
+ runs-on: ${{
+ (matrix.task == 'build-aarch64-gnu' || matrix.task == 'build-aarch64-musl') && 'ubuntu-26.04-arm' ||
+ (matrix.task == 'build-macos-aarch64' || matrix.task == 'clippy-macos-aarch64') && 'macos-14' ||
+ matrix.task == 'build-windows-sdk' && 'windows-latest' ||
+ 'ubuntu-26.04'
+ }}
+ timeout-minutes: ${{ inputs.timeout-minutes }}
+ steps:
+ - uses: actions/checkout@v7.0.1
+
+ # Re-runs replay the original event labels. Read live labels so applying
+ # breaking:storage followed by a re-run can skip an intentional format break.
+ - name: Resolve breaking:storage escape hatch
+ id: storage_hatch
+ if: matrix.task == 'test-storage-compat' && github.event.pull_request.number
+ uses: actions/github-script@v9
+ with:
+ script: |
+ let skip = false;
+ try {
+ const labels = await github.paginate(
+ github.rest.issues.listLabelsOnIssue,
+ {
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.payload.pull_request.number,
+ per_page: 100,
+ },
+ );
+ skip = labels.some(l => l.name === 'breaking:storage');
+ } catch (e) {
+ core.warning(`listLabelsOnIssue failed (${e.status ?? e.code ?? 'network'}), running the check`);
+ }
+ core.info(`breaking:storage escape hatch: skip=${skip}`);
+ core.setOutput('skip', String(skip));
+
+ - name: Run Rust check
+ if: steps.storage_hatch.outputs.skip != 'true'
+ uses: ./.github/actions/rust/pre-merge
+ with:
+ task: ${{ matrix.task }}
+ component: ${{ matrix.component }}
+
+ - name: Upload reports
+ if: always()
+ uses: actions/upload-artifact@v7
+ with:
+ name: ${{ matrix.component }}-${{ matrix.task }}-reports-${{ github.run_id }}-${{ github.run_attempt }}
+ path: reports/**
+ if-no-files-found: ignore
+ retention-days: 7
+
+ test-build:
+ name: Build test artifacts
+ if: ${{ fromJSON(inputs.plan).tests }}
+ runs-on: ubuntu-26.04
+ timeout-minutes: ${{ inputs.timeout-minutes }}
+ steps:
+ - uses: actions/checkout@v7.0.1
+
+ - name: Build coverage test archive
+ uses: ./.github/actions/rust/pre-merge
+ with:
+ task: test-build
+ test-scope: ${{ inputs.test-scope }}
+ save-cache: ${{ inputs.save-cache }}
+ free-disk-space-aggressive: ${{ inputs.test-scope == 'workspace' }}
+
+ - name: Upload test artifacts
+ uses: actions/upload-artifact@v7
+ with:
+ name: rust-test-build-${{ github.run_id }}
+ path: ${{ runner.temp }}/rust-test-build/
+ compression-level: 0
+ if-no-files-found: error
+ overwrite: true
+ retention-days: 7
+
+ tests:
+ name: Test shard ${{ matrix.partition }}/${{ strategy.job-total }}
+ needs: test-build
+ strategy:
+ fail-fast: false
+ matrix:
+ partition: [1, 2, 3]
+ runs-on: ubuntu-26.04
+ timeout-minutes: ${{ inputs.timeout-minutes }}
+ steps:
+ - uses: actions/checkout@v7.0.1
+
+ - name: Download test artifacts
+ uses: actions/download-artifact@v8
+ with:
+ name: rust-test-build-${{ github.run_id }}
+ path: ${{ runner.temp }}/rust-test-build
+
+ - name: Run test shard
+ uses: ./.github/actions/rust/pre-merge
+ with:
+ task: test
+ partition: ${{ matrix.partition }}
+ partitions: ${{ strategy.job-total }}
+
+ - name: Generate coverage report
+ if: always() && !cancelled()
+ run: |
+ if compgen -G 'target/*.profraw' > /dev/null; then
+ cargo llvm-cov report --codecov --output-path codecov.json
+ else
+ echo "::notice::No coverage profiles found; skipping report"
+ fi
+ shell: bash
+
+ - name: Upload coverage to Codecov
+ if: always() && !cancelled() && hashFiles('codecov.json') != ''
+ uses: codecov/codecov-action@v7.1.1
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
+ files: codecov.json
+ disable_search: true
+ flags: rust
+ fail_ci_if_error: false
+ verbose: true
+ override_pr: ${{ github.event.pull_request.number }}
+
+ - name: Upload reports
+ if: always()
+ uses: actions/upload-artifact@v7
+ with:
+ name: rust-test-${{ matrix.partition }}-reports-${{ github.run_id }}-${{ github.run_attempt }}
+ path: |
+ reports/**
+ target/llvm-cov/**
+ coverage.lcov
+ codecov.json
+ if-no-files-found: ignore
+ retention-days: 7
diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml
index 7fb5ff909..0d24d4da2 100644
--- a/.github/workflows/_test.yml
+++ b/.github/workflows/_test.yml
@@ -33,7 +33,6 @@ on:
permissions:
contents: read
- issues: read # listLabelsOnIssue, for the breaking:storage escape hatch
security-events: write
pull-requests: write
@@ -41,10 +40,7 @@ jobs:
run:
# Select runner based on build target
runs-on: ${{
- (inputs.task == 'build-aarch64-gnu' || inputs.task == 'build-aarch64-musl') && 'ubuntu-26.04-arm' ||
- (inputs.task == 'build-macos-aarch64' || inputs.task == 'clippy-macos-aarch64') && 'macos-14' ||
inputs.task == 'build-macos' && 'macos-15' ||
- inputs.task == 'build-windows-sdk' && 'windows-latest' ||
'ubuntu-26.04'
}}
timeout-minutes: 60
@@ -56,75 +52,6 @@ jobs:
if: inputs.component == 'noop'
run: echo "No changes detected, skipping tests"
- # Rust
- # `breaking:storage` is the documented escape hatch for a deliberate
- # on-disk format change, which the compat check cannot pass by
- # construction. The label has to be read live: pre-merge.yml does not
- # subscribe to `labeled` (pr-triage-apply.yml writes S-* labels, so every
- # triage write would re-run the whole gate), and a re-run replays the
- # webhook-frozen `pull_request.labels` from the last push. The operator
- # flow is "check fails, apply label, re-run", which the frozen payload
- # cannot see. Paginated so a PR with more than 100 labels still resolves.
- - name: Resolve breaking:storage escape hatch
- id: storage_hatch
- if: >-
- startsWith(inputs.component, 'rust') &&
- inputs.task == 'test-storage-compat' &&
- github.event.pull_request.number
- uses: actions/github-script@v9
- with:
- script: |
- let skip = false;
- try {
- const labels = await github.paginate(
- github.rest.issues.listLabelsOnIssue,
- {
- owner: context.repo.owner,
- repo: context.repo.repo,
- issue_number: context.payload.pull_request.number,
- per_page: 100,
- },
- );
- skip = labels.some(l => l.name === 'breaking:storage');
- } catch (e) {
- // Fail toward running. A rate limit or a narrowed token must not
- // wave an on-disk format break through as a green check.
- core.warning(`listLabelsOnIssue failed (${e.status ?? e.code ?? 'network'}), running the check`);
- }
- core.info(`breaking:storage escape hatch: skip=${skip}`);
- core.setOutput('skip', String(skip));
-
- - name: Run Rust task
- # Skipping the step (not the job) keeps the leg green so the pre-merge
- # status roll-up stays reportable instead of pending.
- if: >-
- startsWith(inputs.component, 'rust') &&
- steps.storage_hatch.outputs.skip != 'true'
- uses: ./.github/actions/rust/pre-merge
- with:
- task: ${{ inputs.task }}
- component: ${{ inputs.component }}
-
- - name: Upload coverage to Codecov
- # test-storage-compat runs its own nextest invocation without llvm-cov,
- # so it emits no codecov.json. Uploading an empty report under the
- # `rust` flag would read as a coverage drop on the PR. A test task that
- # Cargo-Rail found no affected tests for writes no report either.
- if: >-
- startsWith(inputs.component, 'rust') &&
- startsWith(inputs.task, 'test-') &&
- inputs.task != 'test-storage-compat' &&
- hashFiles('codecov.json') != ''
- uses: codecov/codecov-action@v7.1.1
- with:
- token: ${{ secrets.CODECOV_TOKEN }}
- files: codecov.json
- disable_search: true
- flags: rust
- fail_ci_if_error: false
- verbose: true
- override_pr: ${{ github.event.pull_request.number }}
-
# Python SDK
- name: Set up Docker Buildx for Python
if: inputs.component == 'sdk-python' && inputs.task == 'test'
@@ -283,8 +210,7 @@ jobs:
uses: actions/setup-node@v7
with:
node-version: "24"
- cache: npm
- cache-dependency-path: web/package-lock.json
+ package-manager-cache: false
- name: Run Web UI task
if: inputs.component == 'web-ui'
diff --git a/.github/workflows/_test_bdd.yml b/.github/workflows/_test_bdd.yml
index 945f58054..0e713c05d 100644
--- a/.github/workflows/_test_bdd.yml
+++ b/.github/workflows/_test_bdd.yml
@@ -27,6 +27,14 @@ on:
type: string
required: true
description: "Task to run (e.g., 'bdd-rust', 'bdd-python', etc.)"
+ timeout-minutes:
+ type: number
+ default: 60
+ description: "Job timeout in minutes"
+
+ secrets:
+ CODECOV_TOKEN:
+ required: false
permissions:
contents: read
@@ -35,7 +43,7 @@ permissions:
jobs:
run:
runs-on: ubuntu-26.04
- timeout-minutes: 60
+ timeout-minutes: ${{ inputs.timeout-minutes }}
steps:
- name: Checkout code
uses: actions/checkout@v7.0.1
@@ -44,9 +52,11 @@ jobs:
if: inputs.component == 'noop'
run: echo "No changes detected, skipping tests"
- - name: Setup Rust with cache for BDD
+ - name: Setup Rust for BDD
if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-')
- uses: ./.github/actions/utils/setup-rust-with-cache
+ uses: ./.github/actions/utils/setup-rust
+ with:
+ coverage: ${{ inputs.task == 'bdd-rust' }}
- name: Build server for BDD tests
if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-')
@@ -88,7 +98,11 @@ jobs:
export IGGY_CLI_PATH="target/debug/iggy"
export IGGY_ROOT_USERNAME="iggy"
export IGGY_ROOT_PASSWORD="iggy"
- ./scripts/run-bdd-tests.sh "$SDK_NAME"
+ COVERAGE_ARGS=()
+ if [[ "$SDK_NAME" == rust ]]; then
+ COVERAGE_ARGS=(--coverage)
+ fi
+ ./scripts/run-bdd-tests.sh "${COVERAGE_ARGS[@]}" "$SDK_NAME"
- name: Clean up Docker resources (BDD)
if: always() && startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-')
@@ -96,6 +110,26 @@ jobs:
./scripts/run-bdd-tests.sh clean || true
docker system prune -f || true
+ - name: Generate Rust coverage report
+ if: always() && !cancelled() && inputs.task == 'bdd-rust'
+ run: |
+ if compgen -G 'target/*.profraw' > /dev/null; then
+ cargo llvm-cov report --codecov --output-path codecov.json
+ else
+ echo "::notice::No coverage profiles found; skipping report"
+ fi
+
+ - name: Upload Rust coverage to Codecov
+ if: always() && !cancelled() && inputs.task == 'bdd-rust' && hashFiles('codecov.json', 'reports/rust-bdd-coverage.lcov') != ''
+ uses: codecov/codecov-action@v7.1.1
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
+ files: codecov.json,reports/rust-bdd-coverage.lcov
+ disable_search: true
+ flags: rust-bdd
+ fail_ci_if_error: false
+ override_pr: ${{ github.event.pull_request.number }}
+
- name: Upload reports
if: always()
uses: actions/upload-artifact@v7
@@ -105,5 +139,6 @@ jobs:
reports/**
target/llvm-cov/**
coverage.lcov
+ codecov.json
if-no-files-found: ignore
retention-days: 7
diff --git a/.github/workflows/_test_examples.yml b/.github/workflows/_test_examples.yml
index 024961914..6b3de5130 100644
--- a/.github/workflows/_test_examples.yml
+++ b/.github/workflows/_test_examples.yml
@@ -27,6 +27,7 @@ on:
type: string
required: true
description: "Task to run (e.g., 'examples-rust', 'examples-go', 'examples-node')"
+
permissions:
contents: read
security-events: write
@@ -43,28 +44,17 @@ jobs:
if: inputs.component == 'noop'
run: echo "No changes detected, skipping tests"
- - name: Setup Rust with cache for examples
+ - name: Setup Rust for examples
if: startsWith(inputs.component, 'examples-')
- uses: ./.github/actions/utils/setup-rust-with-cache
+ uses: ./.github/actions/utils/setup-rust
- - name: Setup Node with cache for examples
+ - name: Setup Node for examples
if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-node'
- uses: ./.github/actions/utils/setup-node-with-cache
+ uses: ./.github/actions/utils/setup-node
with:
node-version: "23"
- cache-dependency-path: examples/node/package-lock.json
workspace: examples/node
- - name: Setup Python
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python'
- uses: actions/setup-python@v7.0.0
- with:
- python-version: "3.11"
-
- - name: Setup uv
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python'
- uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
-
- name: Install PHP build dependencies
if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-php'
shell: bash
@@ -91,22 +81,15 @@ jobs:
"${php_bin}" --version
"${php_config}" --version
- - name: Cache uv
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python'
- uses: actions/cache@v6.1.0
- with:
- path: ~/.cache/uv
- key: uv-${{ runner.os }}-${{ hashFiles('examples/python/uv.lock', 'foreign/python/uv.lock', 'bdd/python/uv.lock') }}
-
- - name: Setup Java with cache for examples
+ - name: Setup Java for examples
if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-java'
- uses: ./.github/actions/utils/setup-java-with-cache
+ uses: ./.github/actions/utils/setup-java
with:
gradle-cache-disabled: "true"
- name: Setup Go for examples
if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-go'
- uses: ./.github/actions/utils/setup-go-with-cache
+ uses: ./.github/actions/utils/setup-go
with:
download-deps: "false"
@@ -130,12 +113,11 @@ jobs:
# against the VSR server.
SERVER_BIN="iggy-server"
echo "Building ${SERVER_BIN}..."
- cargo build --locked --bin "${SERVER_BIN}"
-
+ BUILD_ARGS=(--bin "$SERVER_BIN")
if [[ "${{ inputs.task }}" == "examples-rust" ]]; then
- echo "Building additional binaries for Rust examples..."
- cargo build --locked --bin iggy --examples
+ BUILD_ARGS+=(--bin iggy --examples)
fi
+ cargo build --locked "${BUILD_ARGS[@]}"
# Verify server binary was built (needed by all examples)
echo "Verifying server binary:"
@@ -160,6 +142,24 @@ jobs:
fi
fi
+ # Keep Python's PKG_CONFIG_PATH out of the server build above.
+ - name: Setup Python
+ if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python'
+ uses: actions/setup-python@v7.0.0
+ with:
+ python-version: "3.11"
+
+ - name: Setup uv
+ if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python'
+ uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
+
+ - name: Cache uv
+ if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python'
+ uses: actions/cache@v6.1.0
+ with:
+ path: ~/.cache/uv
+ key: uv-${{ runner.os }}-${{ hashFiles('examples/python/uv.lock', 'foreign/python/uv.lock', 'bdd/python/uv.lock') }}
+
- name: Build PHP extension for examples
if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-php'
shell: bash
@@ -218,9 +218,6 @@ jobs:
uses: actions/upload-artifact@v7
with:
name: ${{ inputs.component }}-${{ inputs.task }}-reports-${{ github.run_id }}-${{ github.run_attempt }}
- path: |
- reports/**
- target/llvm-cov/**
- coverage.lcov
+ path: reports/**
if-no-files-found: ignore
retention-days: 7
diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml
index 5cd48bef4..78962fc16 100644
--- a/.github/workflows/coverage-baseline.yml
+++ b/.github/workflows/coverage-baseline.yml
@@ -83,87 +83,48 @@ jobs:
name: Rust coverage baseline
needs: detect
if: needs.detect.outputs.rust == 'true'
- runs-on: ubuntu-26.04
- timeout-minutes: 45
- steps:
- - uses: actions/checkout@v7.0.1
-
- - name: Install system dependencies
- run: |
- ./scripts/ci/apt-install.sh gnome-keyring keyutils dbus-x11 libsecret-tools
- rm -f "$HOME"/.local/share/keyrings/*
-
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
- with:
- install-nextest: "true"
- # Also warms the GitHub Actions build cache for subsequent PR builds
- save-cache: "true"
- # llvm-cov instrumentation roughly doubles object sizes; the light
- # cleanup leaves too little headroom (build hit "No space left on
- # device" at ~34 MB free).
- free-disk-space-aggressive: "true"
-
- - name: Install cargo-llvm-cov
- uses: taiki-e/install-action@v2.87.22
- with:
- tool: cargo-llvm-cov
-
- # Doris 4.0.3's start_be.sh hard-`exit 1`s unless vm.max_map_count >= 2000000.
- # Host-only kernel param (no container can raise it). Mirrors the same step in
- # .github/actions/rust/pre-merge/action.yml; without it every Doris connector
- # integration test panics at fixture setup.
- - name: Raise vm.max_map_count for Doris
- run: sudo sysctl -w vm.max_map_count=2000000 || true
- shell: bash
-
- - name: Build and test with coverage
- run: |
- source <(cargo llvm-cov show-env --export-prefix)
- cargo build --locked
- cargo test --locked --no-run
-
- # Start D-Bus and unlock keyring right before test execution to avoid
- # gnome-keyring auto-locking the collection during the build phase.
- # Previously this ran before `cargo build`, leaving a 7+ minute idle
- # window that triggered org.freedesktop.Secret.Error.IsLocked ~10% of runs.
- eval $(dbus-launch --sh-syntax)
- export DBUS_SESSION_BUS_ADDRESS
- eval $(echo -n "test" | gnome-keyring-daemon --unlock --components=secrets)
- echo -n "warmup" | secret-tool store --label="ci-warmup" ci-test warmup
-
- cargo nextest run --locked --no-fail-fast --profile ci
- shell: bash
-
- - name: Generate coverage report
- run: |
- source <(cargo llvm-cov show-env --export-prefix)
- cargo llvm-cov report --codecov --output-path codecov.json
- shell: bash
-
- - name: Upload to Codecov
- uses: codecov/codecov-action@v7.1.1
- with:
- token: ${{ secrets.CODECOV_TOKEN }}
- files: codecov.json
- disable_search: true
- flags: rust
- fail_ci_if_error: false
+ permissions:
+ contents: read
+ issues: read
+ uses: ./.github/workflows/_rust.yml
+ with:
+ plan: '{"checks":[],"tests":true}'
+ test-scope: workspace
+ timeout-minutes: 120
+ save-cache: true
+ build-metadata: true
+ secrets:
+ CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+ rust-bdd-coverage:
+ name: Rust BDD coverage baseline
+ needs: detect
+ if: needs.detect.outputs.rust == 'true'
+ permissions:
+ contents: read
+ security-events: write
+ uses: ./.github/workflows/_test_bdd.yml
+ with:
+ component: bdd-rust
+ task: bdd-rust
+ timeout-minutes: 120
+ secrets:
+ CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
java-coverage:
name: Java coverage baseline
needs: detect
if: needs.detect.outputs.java == 'true'
runs-on: ubuntu-26.04
- timeout-minutes: 30
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v7.0.1
- - name: Setup Java with cache
- uses: ./.github/actions/utils/setup-java-with-cache
+ - name: Setup Java
+ uses: ./.github/actions/utils/setup-java
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
with:
save-cache: "false"
@@ -200,7 +161,7 @@ jobs:
needs: detect
if: needs.detect.outputs.csharp == 'true'
runs-on: ubuntu-26.04
- timeout-minutes: 30
+ timeout-minutes: 120
strategy:
fail-fast: false
matrix:
@@ -215,8 +176,8 @@ jobs:
8.0.x
10.0.x
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
with:
save-cache: "false"
@@ -285,20 +246,41 @@ jobs:
needs: detect
if: needs.detect.outputs.python == 'true'
runs-on: ubuntu-26.04
- timeout-minutes: 30
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v7.0.1
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
+ with:
+ save-cache: "false"
+
+ # setup-python changes PKG_CONFIG_PATH and invalidates cached native server dependencies.
+ - name: Build server Docker image for TLS tests
+ run: |
+ # test_tls.py spawns the server in a container; build it from the
+ # same binaries the plain-TCP tests run against.
+ cargo build --locked --bin iggy-server --bin iggy
+ docker build \
+ -f core/server/Dockerfile \
+ --target runtime-prebuilt \
+ --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \
+ --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \
+ -t iggy-server:local .
+ shell: bash
+
+ - name: Start Iggy server
+ id: iggy
+ uses: ./.github/actions/utils/server-start
+ with:
+ mode: bin
+ bin: target/debug/iggy-server
+
- name: Setup Python
uses: actions/setup-python@v7.0.0
with:
python-version: "3.11"
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
- with:
- save-cache: "false"
-
- name: Install cargo-llvm-cov
uses: taiki-e/install-action@v2.87.22
with:
@@ -326,25 +308,6 @@ jobs:
uv run --no-sync maturin develop
shell: bash
- - name: Build server Docker image for TLS tests
- run: |
- # test_tls.py spawns the server in a container; build it from the
- # same binaries the plain-TCP tests run against.
- cargo build --locked --bin iggy-server --bin iggy
- docker build \
- -f core/server/Dockerfile \
- --target runtime-prebuilt \
- --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \
- --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \
- -t iggy-server:local .
- shell: bash
-
- - name: Start Iggy server
- id: iggy
- uses: ./.github/actions/utils/server-start
- with:
- cargo-bin: iggy-server
-
- name: Run tests
run: |
cd foreign/python
@@ -409,7 +372,7 @@ jobs:
needs: detect
if: needs.detect.outputs.php == 'true'
runs-on: ubuntu-26.04
- timeout-minutes: 30
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v7.0.1
@@ -432,7 +395,7 @@ jobs:
needs: detect
if: needs.detect.outputs.node == 'true'
runs-on: ubuntu-26.04
- timeout-minutes: 30
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v7.0.1
@@ -440,14 +403,13 @@ jobs:
uses: actions/setup-node@v7
with:
node-version: "23"
- cache: "npm"
- cache-dependency-path: foreign/node/package-lock.json
+ package-manager-cache: false
- name: Pin npm to 11 to match Dependabot lockfile format
run: npm install -g npm@11
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
with:
save-cache: "false"
@@ -499,15 +461,15 @@ jobs:
needs: detect
if: needs.detect.outputs.go == 'true'
runs-on: ubuntu-26.04
- timeout-minutes: 30
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v7.0.1
- name: Setup Go
- uses: ./.github/actions/utils/setup-go-with-cache
+ uses: ./.github/actions/utils/setup-go
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
with:
save-cache: "false"
@@ -587,7 +549,7 @@ jobs:
needs: detect
if: needs.detect.outputs.cpp == 'true'
runs-on: ubuntu-26.04
- timeout-minutes: 45
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v7.0.1
@@ -608,17 +570,14 @@ jobs:
warm-cache-macos:
name: Warm macOS cache
needs: detect
- # Gated on the rust key: the cache goes stale only when Rust sources
- # change, and the Swatinem key is Cargo.lock-based, so skipping the save
- # on non-Rust pushes leaves the previous cache valid.
if: needs.detect.outputs.rust == 'true'
runs-on: macos-14
- timeout-minutes: 30
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v7.0.1
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
+ - name: Setup Rust
+ uses: ./.github/actions/utils/setup-rust
with:
save-cache: "true"
diff --git a/.github/workflows/pr-skill-review-run.yml b/.github/workflows/pr-skill-review-run.yml
index 28d48c327..96286c04a 100644
--- a/.github/workflows/pr-skill-review-run.yml
+++ b/.github/workflows/pr-skill-review-run.yml
@@ -231,6 +231,8 @@ jobs:
- name: Setup Node
if: steps.parse.outputs.status == 'ok'
uses: actions/setup-node@v7
+ with:
+ package-manager-cache: false
- name: Install the agent
if: steps.parse.outputs.status == 'ok'
diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml
index 24c0017f1..bcb96cf60 100644
--- a/.github/workflows/pre-merge.yml
+++ b/.github/workflows/pre-merge.yml
@@ -55,16 +55,12 @@ jobs:
# Rust components
test-rust:
- name: Rust • ${{ matrix.component }}/${{ matrix.task }}
+ name: Rust
needs: detect
- if: ${{ fromJson(needs.detect.outputs.rust_matrix).include[0].component != 'noop' }}
- strategy:
- fail-fast: true
- matrix: ${{ fromJson(needs.detect.outputs.rust_matrix) }}
- uses: ./.github/workflows/_test.yml
+ if: needs.detect.outputs.rust_plan != ''
+ uses: ./.github/workflows/_rust.yml
with:
- component: ${{ matrix.component }}
- task: ${{ matrix.task }}
+ plan: ${{ needs.detect.outputs.rust_plan }}
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
@@ -211,6 +207,8 @@ jobs:
with:
component: ${{ matrix.component }}
task: ${{ matrix.task }}
+ secrets:
+ CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
# Examples Tests
test-examples:
@@ -237,10 +235,11 @@ jobs:
uses: actions/github-script@v9
with:
script: |
- const jobs = await github.rest.actions.listJobsForWorkflowRun({
+ const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
owner: context.repo.owner,
repo: context.repo.repo,
- run_id: context.runId
+ run_id: context.runId,
+ per_page: 100
});
const jobTimes = {};
@@ -263,7 +262,7 @@ jobs:
// Log job names for debugging
console.log('Job names found:');
- for (const job of jobs.data.jobs) {
+ for (const job of jobs) {
console.log(` - ${job.name}: ${job.status} (conclusion: ${job.conclusion || 'N/A'})`);
jobStatuses[job.name] = job.conclusion || job.status;
@@ -305,7 +304,6 @@ jobs:
};
// Set outputs for each component
- const rust = findJobInfo('Rust •');
const python = findJobInfo('Python •');
const php = findJobInfo('PHP •');
const node = findJobInfo('Node •');
@@ -327,7 +325,13 @@ jobs:
: { time: null, status: 'skipped' };
// Output formatted durations
- core.setOutput('rust_time', formatJobDuration(rust));
+ const rustJobs = jobs.filter(job => job.name.startsWith('Rust /') &&
+ job.conclusion !== 'skipped' && job.started_at && job.completed_at);
+ const rustDuration = rustJobs.length ? formatDuration(
+ Math.max(...rustJobs.map(job => new Date(job.completed_at).getTime())) -
+ Math.min(...rustJobs.map(job => new Date(job.started_at).getTime()))
+ ) : '-';
+ core.setOutput('rust_time', rustDuration);
core.setOutput('python_time', formatJobDuration(python));
core.setOutput('php_time', formatJobDuration(php));
core.setOutput('node_time', formatJobDuration(node));
@@ -345,7 +349,7 @@ jobs:
// Calculate total time - find the earliest job start time
let earliestStart = null;
let latestEnd = null;
- for (const job of jobs.data.jobs) {
+ for (const job of jobs) {
if (job.started_at) {
const start = new Date(job.started_at);
if (!earliestStart || start < earliestStart) {
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 425f2b91e..15705dfaa 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -1040,7 +1040,7 @@ jobs:
# need the toolchain.
- name: Setup Rust toolchain (if needed)
if: matrix.type == 'python'
- uses: ./.github/actions/utils/setup-rust-with-cache
+ uses: ./.github/actions/utils/setup-rust
- name: Debug matrix
run: echo '${{ toJson(matrix) }}'
diff --git a/bdd/docker-compose.cluster.yml b/bdd/docker-compose.cluster.yml
index fb9b02954..0ccc3a676 100644
--- a/bdd/docker-compose.cluster.yml
+++ b/bdd/docker-compose.cluster.yml
@@ -52,6 +52,7 @@ x-cluster-node: &cluster-node
hard: -1
x-cluster-topology: &cluster-topology
+ LLVM_PROFILE_FILE: ${BDD_SERVER_PROFILE_FILE:-}
IGGY_ROOT_USERNAME: iggy
IGGY_ROOT_PASSWORD: iggy
IGGY_CLUSTER_ENABLED: "true"
@@ -116,6 +117,7 @@ services:
ipv4_address: 172.28.0.101
volumes:
- iggy_leader_data:/app/local_data_leader
+ - ../target:/profiles
iggy-follower:
<<: *cluster-node
@@ -142,6 +144,7 @@ services:
ipv4_address: 172.28.0.102
volumes:
- iggy_follower_data:/app/local_data_follower
+ - ../target:/profiles
rust-bdd:
<<: *cluster-bdd-deps
diff --git a/bdd/docker-compose.coverage.yml b/bdd/docker-compose.coverage.yml
index 79ea67d3b..e1c58fea8 100644
--- a/bdd/docker-compose.coverage.yml
+++ b/bdd/docker-compose.coverage.yml
@@ -23,20 +23,25 @@
services:
rust-bdd:
+ build:
+ args:
+ COVERAGE: "true"
volumes:
- ../reports:/reports
command:
- - sh
+ - bash
+ - -euo
+ - pipefail
- -c
- |
- rustup component add llvm-tools-preview \
- && cargo install cargo-llvm-cov \
- && case "$$BDD_FEATURE" in
- basic_messaging) cargo llvm-cov test -p bdd --features "$$BDD_RUST_FEATURES" --test basic_messaging --lcov --output-path /reports/rust-bdd-coverage.lcov ;;
- leader_redirection) cargo llvm-cov test -p bdd --features "$$BDD_RUST_FEATURES" --test leader_redirection --lcov --output-path /reports/rust-bdd-coverage.lcov ;;
- stream_crud) cargo llvm-cov test -p bdd --features "$$BDD_RUST_FEATURES" --test stream_crud --lcov --output-path /reports/rust-bdd-coverage.lcov ;;
- *) cargo llvm-cov test -p bdd --features "$$BDD_RUST_FEATURES" --lcov --output-path /reports/rust-bdd-coverage.lcov ;;
+ coverage_env=$$(cargo llvm-cov show-env --sh --remap-path-prefix)
+ eval "$$coverage_env"
+ trap 'cargo llvm-cov report --lcov --output-path /reports/rust-bdd-coverage.lcov' EXIT
+ TEST_ARGS=()
+ case "$$BDD_FEATURE" in
+ basic_messaging|leader_redirection|raw_command|stream_crud) TEST_ARGS=(--test "$$BDD_FEATURE") ;;
esac
+ cargo test --locked -p bdd --features "$$BDD_RUST_FEATURES" "$${TEST_ARGS[@]}"
python-bdd:
volumes:
diff --git a/bdd/docker-compose.server.yml b/bdd/docker-compose.server.yml
index 88868ace3..dcda9539e 100644
--- a/bdd/docker-compose.server.yml
+++ b/bdd/docker-compose.server.yml
@@ -60,6 +60,7 @@ services:
retries: 30
start_period: 2s
environment:
+ - LLVM_PROFILE_FILE=${BDD_SERVER_PROFILE_FILE:-}
- RUST_LOG=info
- IGGY_ROOT_USERNAME=iggy
- IGGY_ROOT_PASSWORD=iggy
@@ -71,6 +72,7 @@ services:
- IGGY_WEBSOCKET_ADDRESS=0.0.0.0:8070
volumes:
- iggy_data:/app/local_data
+ - ../target:/profiles
networks:
- iggy-bdd-network
diff --git a/bdd/docker-compose.yml b/bdd/docker-compose.yml
index 01dee9292..37a460ca8 100644
--- a/bdd/docker-compose.yml
+++ b/bdd/docker-compose.yml
@@ -21,6 +21,8 @@ services:
build:
context: ..
dockerfile: bdd/rust/Dockerfile
+ args:
+ BDD_RUST_FEATURES: ${BDD_RUST_FEATURES:-bdd}
environment:
- IGGY_ROOT_USERNAME=iggy
- IGGY_ROOT_PASSWORD=iggy
diff --git a/bdd/rust/Dockerfile b/bdd/rust/Dockerfile
index ff967433d..1ffa83ea2 100644
--- a/bdd/rust/Dockerfile
+++ b/bdd/rust/Dockerfile
@@ -18,22 +18,44 @@
ARG RUST_VERSION=1.99
FROM rust:${RUST_VERSION}-slim-trixie
+ARG COVERAGE=false
+ARG BDD_RUST_FEATURES=bdd
+ARG LLVM_COV_VERSION=0.9.1
+
+ENV RUSTFLAGS="-C linker=gcc" \
+ CARGO_INCREMENTAL=0 \
+ CARGO_PROFILE_DEV_DEBUG=0 \
+ CARGO_PROFILE_TEST_DEBUG=0
+
+SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
WORKDIR /app
RUN apt-get update && apt-get install -y \
+ ca-certificates \
+ curl \
libhwloc-dev \
libudev-dev \
pkg-config \
&& rm -rf /var/lib/apt/lists/*
-# Copy everything
-COPY . .
+RUN if [[ "$COVERAGE" == true ]]; then \
+ rustup component add llvm-tools-preview; \
+ host=$(rustc -vV | sed -n 's/^host: //p'); \
+ curl -fsSL --retry 5 --retry-all-errors \
+ "https://github.com/taiki-e/cargo-llvm-cov/releases/download/v${LLVM_COV_VERSION}/cargo-llvm-cov-${host}.tar.gz" \
+ -o /tmp/llvm-cov.tar.gz; \
+ tar -xzf /tmp/llvm-cov.tar.gz -C /usr/local/cargo/bin; \
+ rm /tmp/llvm-cov.tar.gz; \
+ fi
-# Create features directory for BDD feature files
-RUN mkdir -p /app/features
-
-# Build tests
-RUN cargo test --no-run -p bdd
+COPY . .
+# The shared Docker context includes host server binaries for other images.
+# Their coverage mappings belong in the host report, not the BDD report.
+RUN rm -rf target && mkdir -p /app/features
+RUN if [[ "$COVERAGE" == true ]]; then \
+ coverage_env=$(cargo llvm-cov show-env --sh --remap-path-prefix); \
+ eval "$coverage_env"; \
+ fi; \
+ cargo test --locked --no-run -p bdd --features "$BDD_RUST_FEATURES"
-# Default command will be overridden by docker-compose
-CMD ["cargo", "test", "-p", "bdd", "--features", "bdd"]
+CMD ["cargo", "test", "--locked", "-p", "bdd", "--features", "bdd"]
diff --git a/codecov.yml b/codecov.yml
index 2d927b241..f33f0de4e 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -55,6 +55,9 @@ flag_management:
- name: rust
paths:
- core/
+ - name: rust-bdd
+ paths:
+ - core/
- name: java
paths:
- foreign/java/
diff --git a/scripts/ci/coverage-baseline-affected.sh b/scripts/ci/coverage-baseline-affected.sh
index aa1187d8f..3b0f96af8 100755
--- a/scripts/ci/coverage-baseline-affected.sh
+++ b/scripts/ci/coverage-baseline-affected.sh
@@ -75,7 +75,7 @@ changed() {
for gate in "${GATES[@]}"; do
case "$gate" in
# .config holds the nextest and Cargo-Rail policy every Rust test run reads.
- rust) paths=(core gateways Cargo.toml Cargo.lock rust-toolchain.toml .cargo .config) ;;
+ rust) paths=(core gateways bdd/rust bdd/scenarios ':(glob)bdd/docker-compose*.yml' examples/rust Cargo.toml Cargo.lock rust-toolchain.toml .cargo .config) ;;
java) paths=(foreign/java) ;;
csharp) paths=(foreign/csharp) ;;
python) paths=(foreign/python) ;;
diff --git a/scripts/ci/storage-compat.sh b/scripts/ci/storage-compat.sh
index edfabe02c..296893268 100755
--- a/scripts/ci/storage-compat.sh
+++ b/scripts/ci/storage-compat.sh
@@ -243,6 +243,10 @@ else
BASELINE_TARGET_DIR="${WORKTREE_DIR}/target"
(
cd "${WORKTREE_DIR}"
+ # The reference revision must not contribute counters to HEAD's coverage report.
+ if [[ "${__CARGO_LLVM_COV_RUSTC_WRAPPER:-}" == 1 ]]; then
+ export RUSTC_WRAPPER="${__CARGO_LLVM_COV_RUSTC_WRAPPER_PRE_EXISTING:-}"
+ fi
CARGO_TARGET_DIR="${BASELINE_TARGET_DIR}" cargo build --locked -p server --bin iggy-server
)
diff --git a/scripts/run-bdd-tests.sh b/scripts/run-bdd-tests.sh
index acc6f913b..aadd6e2e9 100755
--- a/scripts/run-bdd-tests.sh
+++ b/scripts/run-bdd-tests.sh
@@ -78,7 +78,8 @@ case "$FEATURE" in
esac
if [ "$COVERAGE" = "1" ]; then
COMPOSE_FILES+=(-f docker-compose.coverage.yml)
- mkdir -p ../reports
+ mkdir -p ../reports ../target
+ export BDD_SERVER_PROFILE_FILE="/profiles/bdd-server-%h-%p-%m.profraw"
fi
cleanup(){