You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/10/23 17:10:52 UTC
(iggy) branch master updated: perf(ci): improve cache reuse and reduce Miri workload (#4624)
This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new a30717684 perf(ci): improve cache reuse and reduce Miri workload (#4624)
a30717684 is described below
commit a307176846a90ea01c740c0027e46b47d30eeffb
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Fri Oct 23 19:10:47 2026 +0200
perf(ci): improve cache reuse and reduce Miri workload (#4624)
---
.config/nextest.toml | 17 +++-
.github/actions/java-gradle/pre-merge/action.yml | 40 +++++-----
.github/actions/rust/pre-merge/action.yml | 90 +++++++++++++--------
.github/actions/utils/free-disk-space/action.yml | 19 +----
.github/actions/utils/setup-rust/action.yml | 12 +--
.github/config/components.yml | 3 +-
.github/workflows/_build_ci_artifacts.yml | 1 -
.../src/requests/messages/poll_messages.rs | 91 ---------------------
justfile | 6 +-
scripts/ci/test-runtime-artifacts.py | 93 +++++++++++++++++++++-
10 files changed, 190 insertions(+), 182 deletions(-)
diff --git a/.config/nextest.toml b/.config/nextest.toml
index 80f54f749..cfb54ed76 100644
--- a/.config/nextest.toml
+++ b/.config/nextest.toml
@@ -16,9 +16,10 @@
# under the License.
# Floor for the flags CI and scripts/ci/storage-compat.sh pass: --no-tests
-# (0.9.75), --run-ignored only (0.9.76) and --ignore-default-filter (0.9.77).
+# (0.9.75), --run-ignored only (0.9.76), --ignore-default-filter (0.9.77),
+# and archive -E (0.9.109).
# nextest refuses to run below it; setup-rust installs a pinned version above this floor.
-nextest-version = { required = "0.9.77" }
+nextest-version = { required = "0.9.109" }
[[profile.default.overrides]]
# This is a solution (or actually a workaround) for the problem that nextest does not support
@@ -96,5 +97,17 @@ slow-timeout = { period = "60s", terminate-after = 60 }
[profile.default]
slow-timeout = { period = "60s", terminate-after = 5 }
+[profile.default-miri]
+# Keep Miri bounded by the CI job timeout instead of the native-test limit.
+slow-timeout = { period = "60s" }
+
+[[profile.default-miri.overrides]]
+# Start the options boundary test early so other work overlaps it.
+filter = '''
+ package(=iggy_binary_protocol)
+ & test(=primitives::options::tests::entry_count_over_limit_is_rejected)
+'''
+priority = 100
+
[profile.ci]
retries = 3
diff --git a/.github/actions/java-gradle/pre-merge/action.yml b/.github/actions/java-gradle/pre-merge/action.yml
index 060c3b489..9f4dcb781 100644
--- a/.github/actions/java-gradle/pre-merge/action.yml
+++ b/.github/actions/java-gradle/pre-merge/action.yml
@@ -37,7 +37,8 @@ runs:
working-directory: foreign/java
run: |
echo "::group::Linting foreign/java"
- ./gradlew check -x test
+ TIMEFORMAT='Java SDK lint: %3R seconds'
+ time ./gradlew check -x test
echo "::endgroup::"
- name: Lint examples
@@ -46,7 +47,8 @@ runs:
working-directory: examples/java
run: |
echo "::group::Linting examples/java"
- ./gradlew check -x test
+ TIMEFORMAT='Java examples lint: %3R seconds'
+ time ./gradlew check -x test
echo "::endgroup::"
- name: Lint BDD
@@ -55,7 +57,8 @@ runs:
working-directory: bdd/java
run: |
echo "::group::Linting bdd/java"
- ./gradlew check -x test
+ TIMEFORMAT='Java BDD lint: %3R seconds'
+ time ./gradlew check -x test
echo "::endgroup::"
- name: Build
@@ -63,18 +66,11 @@ runs:
if: inputs.task == 'build'
working-directory: foreign/java
run: |
- ./gradlew build -x test -x checkstyleMain -x checkstyleTest -x spotlessCheck
- BUILD_EXIT_CODE=$?
-
- # List artifacts only if build succeeded
- if [ $BUILD_EXIT_CODE -eq 0 ]; then
- echo ""
- echo "Build artifacts:"
- find . -path "*/build/libs/*.jar" -type f 2>/dev/null | head -20 || echo "No jar artifacts found in build/libs directories"
- fi
-
- # Exit with build exit code
- exit $BUILD_EXIT_CODE
+ TIMEFORMAT='Java build: %3R seconds'
+ time ./gradlew build -x test -x checkstyleMain -x checkstyleTest -x spotlessCheck
+ echo ""
+ echo "Build artifacts:"
+ find . -path "*/build/libs/*.jar" -type f 2>/dev/null | head -20 || echo "No jar artifacts found in build/libs directories"
- name: Download runtime binaries
if: inputs.task == 'test'
@@ -98,7 +94,11 @@ runs:
working-directory: foreign/java
env:
USE_EXTERNAL_SERVER: true
- run: ./gradlew test
+ run: |
+ TIMEFORMAT='Java test compilation: %3R seconds'
+ time ./gradlew testClasses
+ TIMEFORMAT='Java tests: %3R seconds'
+ time ./gradlew test
- name: Check for ByteBuf leaks
if: ${{ !cancelled() && inputs.task == 'test' }}
@@ -116,7 +116,9 @@ runs:
working-directory: foreign/java
env:
USE_EXTERNAL_SERVER: true
- run: ./gradlew jacocoAggregatedReport
+ run: |
+ TIMEFORMAT='Java coverage report: %3R seconds'
+ time ./gradlew jacocoAggregatedReport
- name: Copy coverage reports
if: ${{ !cancelled() && inputs.task == 'test' }}
@@ -182,7 +184,9 @@ runs:
env:
USE_EXTERNAL_SERVER: true
IGGY_TCP_TLS_ENABLED: "true"
- run: ./gradlew :iggy:test --tests "*TlsConnectionTest*"
+ run: |
+ TIMEFORMAT='Java TLS tests: %3R seconds'
+ time ./gradlew :iggy:test --tests "*TlsConnectionTest*"
- name: Stop Iggy server (TLS)
if: always() && inputs.task == 'test'
diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml
index b0d92d388..30bff0c1e 100644
--- a/.github/actions/rust/pre-merge/action.yml
+++ b/.github/actions/rust/pre-merge/action.yml
@@ -40,10 +40,6 @@ inputs:
description: "Save build dependencies for subsequent PRs"
required: false
default: "false"
- free-disk-space-aggressive:
- description: "Use deep disk cleanup for full workspace coverage builds"
- required: false
- default: "false"
runs:
using: "composite"
@@ -51,9 +47,8 @@ runs:
- name: Setup Rust
uses: ./.github/actions/utils/setup-rust
with:
- # Only test jobs execute nextest. Avoid downloading it in lint, metadata,
- # and cross-build jobs.
- install-nextest: ${{ inputs.task == 'test' || startsWith(inputs.task, 'test-') }}
+ # Test and Miri jobs execute nextest.
+ install-nextest: ${{ inputs.task == 'miri' || inputs.task == 'test' || startsWith(inputs.task, 'test-') }}
coverage: ${{ inputs.task == 'runtime' || inputs.task == 'test-build' || inputs.task == 'test' }}
# These tasks do not compile workspace crates and need no native
# libraries. All compiling tasks keep the existing dependency setup.
@@ -70,11 +65,9 @@ runs:
# Shards download the build artifact. Metadata-only tasks need no target cache.
read-cache: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'machete' || inputs.task == 'test') && 'false' || 'true' }}
save-cache: ${{ inputs.save-cache }}
- # Light legs fit in the runner's ~89 GiB default headroom; skip the
- # ~20-45s reclaim. Disk-heavy legs (coverage build + testcontainers
- # images on test jobs, cross-builds, miri, verify-publish) keep it.
- free-disk-space: ${{ (inputs.task == 'runtime' || inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'check-msrv' || inputs.task == 'machete' || inputs.task == 'doctest') && 'false' || 'true' }}
- free-disk-space-aggressive: ${{ inputs.free-disk-space-aggressive }}
+ # Artifact builders do not pull test containers. Keep cleanup on the
+ # test shards and other disk-heavy jobs.
+ free-disk-space: ${{ (inputs.task == 'runtime' || inputs.task == 'test-build' || inputs.task == 'miri' || inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'check-msrv' || inputs.task == 'machete' || inputs.task == 'doctest') && 'false' || 'true' }}
- name: Install cargo-sort
if: inputs.task == 'sort'
@@ -202,8 +195,7 @@ runs:
SELECTOR_DIR="$RUNNER_TEMP/cargo-rail-selectors"
mkdir -p "$SELECTOR_DIR"
- TEST_CARGO_ARGS=()
- TEST_TARGET_ARGS=()
+ TEST_ARCHIVE_ARGS=()
TEST_PACKAGES=()
RUNTIME_PACKAGES=()
if [[ "$TEST_SELECTION" == workspace ]]; then
@@ -215,13 +207,10 @@ runs:
elif [[ "$PLAN_STATUS" == success ]] &&
TEST_SCOPE=$(cargo-rail-action plan cargo-scope "$PLAN_FILE" cargo.test) &&
RUNTIME_SCOPE=$(cargo-rail-action plan cargo-scope "$PLAN_FILE" runtime-artifacts) &&
- cargo-rail-action plan cargo-args "$PLAN_FILE" cargo.test > "$SELECTOR_DIR/test-args" &&
cargo-rail-action plan target-args "$PLAN_FILE" cargo.test > "$SELECTOR_DIR/test-target-args" &&
cargo-rail-action plan package-names "$PLAN_FILE" cargo.test > "$SELECTOR_DIR/test-packages" &&
cargo-rail-action plan package-names "$PLAN_FILE" runtime-artifacts > "$SELECTOR_DIR/runtime-packages"; then
# NUL-delimited selectors preserve package and argument boundaries.
- mapfile -d '' -t TEST_CARGO_ARGS < "$SELECTOR_DIR/test-args"
- mapfile -d '' -t TEST_TARGET_ARGS < "$SELECTOR_DIR/test-target-args"
mapfile -d '' -t TEST_PACKAGES < "$SELECTOR_DIR/test-packages"
mapfile -d '' -t RUNTIME_PACKAGES < "$SELECTOR_DIR/runtime-packages"
else
@@ -235,22 +224,50 @@ runs:
exit 0
fi
+ if [[ "$TEST_SCOPE" == packages ]]; then
+ # Cargo-Rail narrows targets only through --test NAME pairs; otherwise
+ # every test binary in the selected packages remains eligible.
+ ARCHIVE_FILTER=$(jq -nr \
+ --rawfile packages "$SELECTOR_DIR/test-packages" \
+ --rawfile targets "$SELECTOR_DIR/test-target-args" '
+ ($packages | split("\u0000")[:-1] |
+ map("package(=\(.))") | join(" | ")) as $packages |
+ ($targets | split("\u0000")[:-1] | to_entries |
+ map(select(.key % 2 == 1) | "binary(=\(.value))") |
+ join(" | ")) as $targets |
+ if $targets == "" then $packages
+ else "(\($packages)) & kind(test) & (\($targets))" end')
+ TEST_ARCHIVE_ARGS=(-E "$ARCHIVE_FILTER")
+ fi
+
# SDKs need only server/CLI. Keep Rust-only binaries and plugins with their tests.
- BUILD_ARGS=()
+ BUILD_RUNTIME=false
if [[ "$RUNTIME_SCOPE" == workspace ]]; then
- BUILD_ARGS=(--workspace --exclude server --exclude iggy-cli)
+ BUILD_RUNTIME=true
elif [[ "$RUNTIME_SCOPE" == packages ]]; then
for package in "${RUNTIME_PACKAGES[@]}"; do
if [[ "$package" != server && "$package" != iggy-cli ]]; then
- BUILD_ARGS+=(-p "$package")
+ BUILD_RUNTIME=true
+ break
fi
done
fi
: > "$ARTIFACT_DIR/runtime-files"
- if (( ${#BUILD_ARGS[@]} )); then
- cargo build --locked "${BUILD_ARGS[@]}" --message-format=json-render-diagnostics > "$ARTIFACT_DIR/build.jsonl"
- jq -r 'select(.reason == "compiler-artifact") |
+ if [[ "$BUILD_RUNTIME" == true ]]; then
+ # Match the cache producer's dependency features even for affected-package runs.
+ cargo build --locked --workspace --exclude server --exclude iggy-cli \
+ --message-format=json-render-diagnostics > "$ARTIFACT_DIR/build.jsonl"
+ RUNTIME_PACKAGE_IDS='[]'
+ if [[ "$RUNTIME_SCOPE" == packages ]]; then
+ RUNTIME_PACKAGE_IDS=$(cargo metadata --locked --no-deps --format-version 1 |
+ jq --rawfile packages "$SELECTOR_DIR/runtime-packages" '
+ ($packages | split("\u0000")[:-1]) as $names |
+ [.packages[] | select(.name as $name | $names | index($name)) | .id]')
+ fi
+ jq -r --arg scope "$RUNTIME_SCOPE" --argjson packages "$RUNTIME_PACKAGE_IDS" '
+ select(.reason == "compiler-artifact") |
+ select($scope == "workspace" or (.package_id as $id | $packages | index($id))) |
(select(.target.kind | index("bin")) | .executable // empty),
(select(.target.crate_types | index("cdylib")) | .filenames[] | select(endswith(".so")))' \
"$ARTIFACT_DIR/build.jsonl" | sort -u | \
@@ -317,8 +334,9 @@ runs:
tar --zstd -cf "$ARTIFACT_DIR/kafka-fixtures.tar.zst" gateways/kafka/tools/kafka-tool/kafka_messages
fi
- cargo nextest archive --locked --profile ci \
- "${TEST_CARGO_ARGS[@]}" "${TEST_TARGET_ARGS[@]}" \
+ # Filtering the archive preserves the workspace compilation graph.
+ cargo nextest archive --locked --profile ci --workspace \
+ "${TEST_ARCHIVE_ARGS[@]}" \
--archive-file "$ARTIFACT_DIR/nextest.tar.zst"
shell: bash
@@ -422,21 +440,23 @@ runs:
# model and accepts the legitimate `NonNull::add` arithmetic in
# `iobuf.rs` that stacked-borrows would reject. Strict-provenance
# rejects int-to-pointer round trips, which we don't use.
- - name: Install nightly Rust + miri component
- if: inputs.task == 'miri'
- run: |
- rustup toolchain install nightly-2026-10-02 --component miri --profile minimal --no-self-update
- cargo +nightly-2026-10-02 miri setup
- shell: bash
-
- name: Run miri
if: inputs.task == 'miri'
env:
+ MIRI_TOOLCHAIN: nightly-2026-10-23
MIRIFLAGS: "-Zmiri-tree-borrows -Zmiri-strict-provenance"
run: |
- cargo +nightly-2026-10-02 miri test --locked \
- -p iggy_binary_protocol \
- -p consensus
+ rustup toolchain install "$MIRI_TOOLCHAIN" --component miri --profile minimal --no-self-update
+ TIMEFORMAT='Miri sysroot setup: %3R seconds'
+ time cargo +"$MIRI_TOOLCHAIN" miri setup
+ miri_args=(--locked -p iggy_binary_protocol -p consensus)
+ TIMEFORMAT='Miri test compilation: %3R seconds'
+ time cargo +"$MIRI_TOOLCHAIN" miri nextest run --no-run "${miri_args[@]}"
+ TIMEFORMAT='Miri tests: %3R seconds'
+ time cargo +"$MIRI_TOOLCHAIN" miri nextest run "${miri_args[@]}"
+ # Nextest does not run doctests.
+ TIMEFORMAT='Miri doctests: %3R seconds'
+ time cargo +"$MIRI_TOOLCHAIN" miri test --doc "${miri_args[@]}"
shell: bash
# aarch64 builds (run on ARM64 runners)
diff --git a/.github/actions/utils/free-disk-space/action.yml b/.github/actions/utils/free-disk-space/action.yml
index a2e815c1e..f8a7b1973 100644
--- a/.github/actions/utils/free-disk-space/action.yml
+++ b/.github/actions/utils/free-disk-space/action.yml
@@ -21,20 +21,11 @@ description: >-
pre-installed toolchains the Iggy build does not use.
Logs reclaimed GiB and elapsed seconds.
-inputs:
- aggressive:
- description: >-
- When "true", run jlumbroso/free-disk-space for a deep cleanup
- (~30+ GiB) instead of the light built-in removal (~12 GiB).
- Enable only for disk-heavy jobs (e.g. llvm-cov instrumented builds).
- required: false
- default: "false"
-
runs:
using: "composite"
steps:
- name: Cleanup disk space (light)
- if: runner.os == 'Linux' && inputs.aggressive != 'true'
+ if: runner.os == 'Linux'
shell: bash
run: |
echo "Disk space before cleanup:"
@@ -60,11 +51,3 @@ runs:
echo "::notice::Reclaimed ${reclaimed_gib} GiB in $((end - start))s"
echo "Disk space after cleanup:"
df -h
-
- - name: Cleanup disk space (aggressive)
- if: runner.os == 'Linux' && inputs.aggressive == 'true'
- uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
- with:
- # Preserve the .NET SDK when actions/setup-dotnet provisioned it
- # (DOTNET_ROOT set); removing it breaks subsequent dotnet invocations.
- dotnet: ${{ env.DOTNET_ROOT == '' }}
diff --git a/.github/actions/utils/setup-rust/action.yml b/.github/actions/utils/setup-rust/action.yml
index c4f0a2026..92c8932b2 100644
--- a/.github/actions/utils/setup-rust/action.yml
+++ b/.github/actions/utils/setup-rust/action.yml
@@ -46,16 +46,10 @@ inputs:
free-disk-space:
description: >-
Whether to reclaim runner disk before the build. Default "true".
- Set "false" for light legs (fmt/sort/clippy/check/machete/doctest) that
- fit in the ~89 GiB default headroom and should not pay the ~20-45s cost.
+ Artifact builders, Miri and lightweight checks disable cleanup when
+ they fit within the runner's available disk space.
required: false
default: "true"
- free-disk-space-aggressive:
- description: >-
- Forwarded to free-disk-space. When "true", run jlumbroso for a deep
- cleanup. Enable for disk-heavy jobs (e.g. llvm-cov instrumented builds).
- required: false
- default: "false"
runs:
using: "composite"
@@ -63,8 +57,6 @@ runs:
- name: Free runner disk space
if: inputs.free-disk-space == 'true'
uses: ./.github/actions/utils/free-disk-space
- with:
- aggressive: ${{ inputs.free-disk-space-aggressive }}
- name: Install system dependencies (Linux)
if: runner.os == 'Linux' && inputs.install-system-dependencies == 'true'
diff --git a/.github/config/components.yml b/.github/config/components.yml
index c304adb51..31da9737c 100644
--- a/.github/config/components.yml
+++ b/.github/config/components.yml
@@ -110,6 +110,7 @@ components:
paths:
- "core/binary_protocol/**"
- "core/consensus/**"
+ - ".config/nextest.toml"
tasks:
- "miri"
@@ -204,7 +205,7 @@ components:
- "ci-infrastructure"
paths:
- "Dockerfile*"
- - ".config/**" # nextest + cargo-rail config, read only by the test-* tasks
+ - ".config/**" # nextest and cargo-rail configuration
- "!foreign/**" # Exclude foreign SDKs
- "!web/**" # Exclude web UI
tasks:
diff --git a/.github/workflows/_build_ci_artifacts.yml b/.github/workflows/_build_ci_artifacts.yml
index 5ffcabd9c..cb38e371c 100644
--- a/.github/workflows/_build_ci_artifacts.yml
+++ b/.github/workflows/_build_ci_artifacts.yml
@@ -56,7 +56,6 @@ jobs:
task: ${{ inputs.task }}
test-scope: ${{ inputs.test-scope }}
save-cache: ${{ inputs.save-cache }}
- free-disk-space-aggressive: ${{ inputs.test-scope == 'workspace' }}
- name: Upload artifacts
uses: actions/upload-artifact@v7
diff --git a/core/binary_protocol/src/requests/messages/poll_messages.rs b/core/binary_protocol/src/requests/messages/poll_messages.rs
index 6f5194369..9fef37778 100644
--- a/core/binary_protocol/src/requests/messages/poll_messages.rs
+++ b/core/binary_protocol/src/requests/messages/poll_messages.rs
@@ -147,97 +147,6 @@ impl WireDecode for PollMessagesRequest {
mod tests {
use super::*;
- #[test]
- fn given_poll_encodings_when_peeking_progress_should_match_full_decode() {
- let identifiers = [
- WireIdentifier::numeric(u32::MAX),
- WireIdentifier::named("named-é").unwrap(),
- ];
- for consumer_id in &identifiers {
- for consumer in [
- WireConsumer::consumer(consumer_id.clone()),
- WireConsumer::consumer_group(consumer_id.clone()),
- WireConsumer::external_group(consumer_id.clone()),
- ] {
- for stream_id in &identifiers {
- for topic_id in &identifiers {
- for strategy in [
- WirePollingStrategy::offset(0),
- WirePollingStrategy::offset(u64::MAX),
- WirePollingStrategy::timestamp(u64::MAX),
- WirePollingStrategy::first(),
- WirePollingStrategy::last(),
- WirePollingStrategy::next(),
- ] {
- for partition_id in [None, Some(u32::MAX)] {
- for auto_commit in [false, true] {
- for count in [0, u32::MAX] {
- let request = PollMessagesRequest {
- consumer: consumer.clone(),
- stream_id: stream_id.clone(),
- topic_id: topic_id.clone(),
- partition_id,
- strategy: strategy.clone(),
- count,
- auto_commit,
- };
- let mut bytes = request.to_bytes().to_vec();
- assert_eq!(
- PollMessagesRequest::decode_from(&bytes).unwrap(),
- request
- );
- for end in 0..=bytes.len() {
- assert_progress_peek_matches(&bytes[..end]);
- }
- bytes.extend_from_slice(b"trailing\0\xff");
- assert_progress_peek_matches(&bytes);
- }
- }
- }
- }
- }
- }
- }
- }
- }
-
- #[test]
- fn given_mutated_poll_fields_when_peeking_progress_should_match_full_decode() {
- for identifier in [
- WireIdentifier::numeric(1),
- WireIdentifier::named("café").unwrap(),
- ] {
- let request = PollMessagesRequest {
- consumer: WireConsumer::consumer(identifier.clone()),
- stream_id: identifier.clone(),
- topic_id: identifier,
- partition_id: Some(0),
- strategy: WirePollingStrategy::offset(0),
- count: 0,
- auto_commit: false,
- };
- let mut bytes = request.to_bytes().to_vec();
- for offset in 0..bytes.len() {
- let original = bytes[offset];
- for value in u8::MIN..=u8::MAX {
- bytes[offset] = value;
- assert_progress_peek_matches(&bytes);
- }
- bytes[offset] = original;
- }
- }
- }
-
- fn assert_progress_peek_matches(bytes: &[u8]) {
- let decoded = PollMessagesRequest::decode_from(bytes)
- .map(|request| (request.strategy, request.auto_commit));
- assert_eq!(
- PollMessagesRequest::peek_progress(bytes),
- decoded,
- "{bytes:?}"
- );
- }
-
#[test]
fn roundtrip_with_partition() {
let req = PollMessagesRequest {
diff --git a/justfile b/justfile
index f8c3244a2..15cc6ff0f 100644
--- a/justfile
+++ b/justfile
@@ -73,16 +73,16 @@ clean-test-containers:
{{reap_test_containers}}
# Run Miri (UB detector) on the unsafe-heavy crates that don't pull
-# tokio/compio. Mirrors the `miri` task in CI. Pinned to the same nightly
+# tokio/compio. Uses the same crates, flags and pinned nightly
# as `.github/actions/rust/pre-merge/action.yml` so local runs don't drift
# from CI on the next nightly bump - keep these two dates in sync.
# Requires:
#
-# rustup toolchain install nightly-2026-10-02 --component miri
+# rustup toolchain install nightly-2026-10-23 --component miri
#
miri:
MIRIFLAGS="-Zmiri-tree-borrows -Zmiri-strict-provenance" \
- cargo +nightly-2026-10-02 miri test -p iggy_binary_protocol -p consensus
+ cargo +nightly-2026-10-23 miri test -p iggy_binary_protocol -p consensus
server *ARGS:
cargo run --bin iggy-server {{ARGS}}
diff --git a/scripts/ci/test-runtime-artifacts.py b/scripts/ci/test-runtime-artifacts.py
index e153619ca..5a97ea094 100644
--- a/scripts/ci/test-runtime-artifacts.py
+++ b/scripts/ci/test-runtime-artifacts.py
@@ -134,6 +134,27 @@ class RuntimeBuildSelection(unittest.TestCase):
git = self.root / "git"
git.write_text(f"#!/bin/sh\ncase $1 in\n rev-parse) echo {self.revision} ;;\n cat-file) echo 'parent fixture' ;;\n *) exit 1 ;;\nesac\n")
git.chmod(0o755)
+ self.runtime_files = {
+ "iggy-connectors": ("bin", "target/debug/iggy-connectors"),
+ "iggy_connector_stdout_sink": ("cdylib", "target/debug/libiggy_connector_stdout_sink.so"),
+ "iggy-bench-dashboard-server": ("bin", "target/debug/iggy-bench-dashboard-server"),
+ }
+ packages = []
+ artifacts = []
+ for package, (kind, filename) in self.runtime_files.items():
+ package_id = f"path+file://{self.root}/{package}#0.1.0"
+ packages.append({"name": package, "id": package_id})
+ binary = self.root / filename
+ binary.parent.mkdir(parents=True, exist_ok=True)
+ binary.write_text(package)
+ artifacts.append({
+ "reason": "compiler-artifact", "package_id": package_id,
+ "target": {"name": binary.stem, "kind": [kind], "crate_types": [kind]},
+ "executable": str(binary) if kind == "bin" else None,
+ "filenames": [str(binary)],
+ })
+ (self.root / "metadata.json").write_text(json.dumps({"packages": packages}))
+ (self.root / "build.jsonl").write_text("\n".join(json.dumps(artifact) for artifact in artifacts))
cargo = self.root / "cargo"
cargo.write_text("""#!/usr/bin/env python3
import json
@@ -144,6 +165,9 @@ with (Path(os.environ['RUNNER_TEMP']) / 'cargo-calls').open('a') as calls:
calls.write(json.dumps(sys.argv[1:]) + '\\n')
if sys.argv[1:3] == ['nextest', 'archive']:
Path(sys.argv[sys.argv.index('--archive-file') + 1]).touch()
+elif sys.argv[1] in ('build', 'metadata'):
+ output = 'build.jsonl' if sys.argv[1] == 'build' else 'metadata.json'
+ print((Path(os.environ['RUNNER_TEMP']) / output).read_text())
""")
cargo.chmod(0o755)
generator = self.root / "gateways/kafka/scripts/ci-wire-fixtures.sh"
@@ -161,10 +185,16 @@ if sys.argv[1:3] == ['nextest', 'archive']:
self.assertEqual(calls[0], ["build", "--locked", "--workspace", "--exclude", "server",
"--exclude", "iggy-cli", "--message-format=json-render-diagnostics"])
self.assertEqual(calls[1], ["nextest", "archive", "--locked", "--profile", "ci",
+ "--workspace",
"--archive-file", str(artifact / "nextest.tar.zst")])
- self.assertTrue((artifact / "runtime.tar.zst").is_file())
+ self.assert_runtime_archive([filename for _, filename in self.runtime_files.values()])
self.assertEqual((artifact / "kafka-fixtures").read_text(), "true\n")
+ def assert_runtime_archive(self, files):
+ result = run("tar", "--zstd", "-tf", str(self.root / "rust-test-build/runtime.tar.zst"))
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(sorted(result.stdout.splitlines()), sorted(files))
+
def test_planner_failure_builds_all_test_extras_and_tests(self):
self.assert_workspace_fallback()
@@ -172,13 +202,65 @@ if sys.argv[1:3] == ['nextest', 'archive']:
self.env["TEST_SELECTION"] = "workspace"
self.assert_workspace_fallback()
+ def select(self, packages, *, tests=(), runtime_packages=()):
+ outputs = {
+ "cargo-scope:cargo.test": "packages\n" if packages else "skipped\n",
+ "cargo-scope:runtime-artifacts": "packages\n" if runtime_packages else "skipped\n",
+ "package-names:cargo.test": "".join(package + "\0" for package in packages),
+ "package-names:runtime-artifacts": "".join(package + "\0" for package in runtime_packages),
+ "target-args:cargo.test": "".join("--test\0" + test + "\0" for test in tests),
+ }
+ selector = self.root / "cargo-rail-action"
+ selector.write_text("#!/usr/bin/env python3\nimport sys\n"
+ f"outputs = {outputs!r}\n"
+ "sys.stdout.write(outputs[f'{sys.argv[2]}:{sys.argv[4]}'])\n")
+ selector.chmod(0o755)
+ self.env["PLAN_STATUS"] = "success"
+ result = run("bash", "-euo", "pipefail", "-c", self.script, cwd=self.root, env=self.env)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ calls = self.root / "cargo-calls"
+ return [json.loads(line) for line in calls.read_text().splitlines()] if calls.exists() else []
+
+ def test_affected_packages_keep_workspace_builds_and_filter_only_the_archive(self):
+ calls = self.select(["integration", "iggy"],
+ runtime_packages=["iggy-connectors", "iggy_connector_stdout_sink"])
+ self.assertEqual(calls[0], ["build", "--locked", "--workspace", "--exclude", "server",
+ "--exclude", "iggy-cli", "--message-format=json-render-diagnostics"])
+ self.assertEqual(calls[-1], ["nextest", "archive", "--locked", "--profile", "ci",
+ "--workspace", "-E", "package(=integration) | package(=iggy)",
+ "--archive-file", str(self.root / "rust-test-build/nextest.tar.zst")])
+ self.assert_runtime_archive(["target/debug/iggy-connectors",
+ "target/debug/libiggy_connector_stdout_sink.so"])
+ self.assertEqual((self.root / "rust-test-build/scope").read_text(), "packages\n")
+ self.assertEqual((self.root / "rust-test-build/kafka-fixtures").read_text(), "false\n")
+
+ def test_selected_integration_targets_filter_by_package_kind_and_name(self):
+ calls = self.select(["integration", "iggy"], tests=["round_trip", "protocol"])
+ self.assertEqual(len(calls), 1)
+ self.assertIn("--workspace", calls[0])
+ self.assertNotIn("--test", calls[0])
+ self.assertEqual(calls[0][calls[0].index("-E") + 1],
+ "(package(=integration) | package(=iggy)) & kind(test) & "
+ "(binary(=round_trip) | binary(=protocol))")
+
+ def test_server_and_cli_prerequisites_use_the_shared_runtime(self):
+ calls = self.select(["iggy-gateway-kafka"], runtime_packages=["server", "iggy-cli"])
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0][:2], ["nextest", "archive"])
+ self.assert_runtime_archive([])
+ self.assertEqual((self.root / "rust-test-build/kafka-fixtures").read_text(), "true\n")
+
+ def test_skipped_scope_does_not_compile_or_archive(self):
+ self.assertEqual(self.select([]), [])
+ self.assertEqual((self.root / "rust-test-build/scope").read_text(), "skipped\n")
+ self.assertFalse((self.root / "rust-test-build/nextest.tar.zst").exists())
+
def test_selector_failure_discards_partial_package_selection(self):
selector = self.root / "cargo-rail-action"
selector.write_text("""#!/usr/bin/env bash
case "$2:$4" in
cargo-scope:*) printf 'packages\\n' ;;
- cargo-args:*) printf '%s\\0' '-p' 'selected-test-package' ;;
- target-args:*) printf '%s\\0' '--lib' ;;
+ target-args:*) printf '%s\\0' '--test' 'selected-test' ;;
package-names:cargo.test) printf '%s\\0' 'selected-test-package' ;;
*) exit 1 ;;
esac
@@ -209,6 +291,11 @@ class RuntimeSelection(unittest.TestCase):
[(entry["component"], entry["task"])
for entry in json.loads(outputs["other_matrix"])["include"]])
+ def test_nextest_config_change_runs_miri(self):
+ outputs = self.select([".config/nextest.toml"])
+ plan = json.loads(outputs["rust_plan"])
+ self.assertIn("miri", [entry["task"] for entry in plan["checks"]])
+
def test_runtime_restore_change_exercises_real_consumers(self):
outputs = self.select(["scripts/ci/restore-runtime.sh"])
self.assertEqual(outputs["runtime"], "true")