You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by hg...@apache.org on 2026/10/16 14:36:27 UTC
(iggy) branch master updated: perf(ci): share covered runtimes across test jobs (#4560)
This is an automated email from the ASF dual-hosted git repository.
hubcio pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new c95797383 perf(ci): share covered runtimes across test jobs (#4560)
c95797383 is described below
commit c957973838c48bbcfb80aa9e905e6093b6820705
Author: Hubert Gruszecki <h....@gmail.com>
AuthorDate: Fri Oct 16 16:36:21 2026 +0200
perf(ci): share covered runtimes across test jobs (#4560)
---
.github/actions/bdd/pre-merge/action.yml | 108 +++++++
.github/actions/cpp-bazel/pre-merge/action.yml | 6 +-
.github/actions/csharp-dotnet/pre-merge/action.yml | 7 +-
.github/actions/go/pre-merge/action.yml | 12 +-
.github/actions/java-gradle/pre-merge/action.yml | 8 +-
.github/actions/node-npm/pre-merge/action.yml | 8 +-
.github/actions/php/pre-merge/action.yml | 8 +-
.../actions/python-maturin/pre-merge/action.yml | 9 +-
.github/actions/rust/pre-merge/action.yml | 94 +++---
.github/actions/swift/pre-merge/action.yml | 20 +-
.github/actions/utils/download-runtime/action.yml | 59 ++++
.github/actions/utils/server-start/action.yml | 53 +---
.github/config/components.yml | 4 +
.github/workflows/_build_ci_artifacts.yml | 69 +++++
.github/workflows/_detect.yml | 29 +-
.github/workflows/_rust.yml | 111 +-------
.github/workflows/_rust_tests.yml | 90 ++++++
.github/workflows/_test.yml | 114 ++++----
.github/workflows/_test_bdd.yml | 119 +-------
.github/workflows/_test_examples.yml | 101 +++----
.github/workflows/coverage-baseline.yml | 175 +++++++-----
.github/workflows/pre-merge.yml | 315 +++++++--------------
AGENTS.md | 2 +-
scripts/ci/restore-runtime.sh | 35 +++
scripts/ci/storage-compat.sh | 48 ++--
scripts/ci/test-runtime-artifacts.py | 256 +++++++++++++++++
scripts/run-examples-from-readme.sh | 25 +-
27 files changed, 1098 insertions(+), 787 deletions(-)
diff --git a/.github/actions/bdd/pre-merge/action.yml b/.github/actions/bdd/pre-merge/action.yml
new file mode 100644
index 000000000..0f16cbdde
--- /dev/null
+++ b/.github/actions/bdd/pre-merge/action.yml
@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: bdd-pre-merge
+description: Run a BDD suite with the shared covered runtime
+inputs:
+ component:
+ required: true
+ description: BDD component
+ task:
+ required: true
+ description: BDD SDK task
+ codecov-token:
+ required: false
+ description: Codecov upload token
+runs:
+ using: composite
+ steps:
+ - name: Setup Rust for BDD
+ if: inputs.task == 'bdd-rust'
+ uses: ./.github/actions/utils/setup-rust
+ with:
+ coverage: "true"
+ # The suite compiles in Docker. The host only runs the coverage report.
+ free-disk-space: "false"
+ read-cache: "false"
+
+ - name: Download runtime binaries
+ uses: ./.github/actions/utils/download-runtime
+ with:
+ install-system-dependencies: "false"
+
+ - name: Run BDD tests
+ shell: bash
+ run: |
+ # Extract SDK name from task (format: bdd-<sdk>).
+ SDK_NAME=$(echo "${{ inputs.task }}" | sed 's/^bdd-//')
+ export IGGY_SERVER_PATH="target/debug/iggy-server"
+ echo "Server binary location: $(ls -lh target/debug/iggy-server)"
+
+ echo "Running BDD tests for SDK: $SDK_NAME"
+ echo "Current directory: $(pwd)"
+ echo "CLI binary location: $(ls -lh target/debug/iggy)"
+
+ # Export path to the pre-built cli binary (relative to repo root)
+ export IGGY_CLI_PATH="target/debug/iggy"
+ export IGGY_ROOT_USERNAME="iggy"
+ export IGGY_ROOT_PASSWORD="iggy"
+ COVERAGE_ARGS=()
+ if [[ "$SDK_NAME" == rust ]]; then
+ COVERAGE_ARGS=(--coverage)
+ fi
+ ./scripts/run-bdd-tests.sh "${COVERAGE_ARGS[@]}" "$SDK_NAME"
+
+ - name: Clean up Docker resources (BDD)
+ if: always()
+ shell: bash
+ run: |
+ ./scripts/run-bdd-tests.sh clean || true
+ docker system prune -f || true
+
+ - name: Generate Rust coverage report
+ if: always() && !cancelled() && inputs.task == 'bdd-rust'
+ shell: bash
+ run: |
+ if compgen -G 'target/*.profraw' > /dev/null; then
+ cargo llvm-cov report --codecov --output-path codecov.json
+ else
+ echo "::notice::No coverage profiles found; skipping report"
+ fi
+
+ - name: Upload Rust coverage to Codecov
+ if: always() && !cancelled() && inputs.task == 'bdd-rust' && hashFiles('codecov.json', 'reports/rust-bdd-coverage.lcov') != ''
+ uses: codecov/codecov-action@v7.1.1
+ with:
+ token: ${{ inputs.codecov-token }}
+ files: codecov.json,reports/rust-bdd-coverage.lcov
+ disable_search: true
+ flags: rust-bdd
+ fail_ci_if_error: false
+ override_pr: ${{ github.event.pull_request.number }}
+
+ - name: Upload reports
+ if: always()
+ uses: actions/upload-artifact@v7
+ with:
+ name: ${{ inputs.component }}-${{ inputs.task }}-reports-${{ github.run_id }}-${{ github.run_attempt }}
+ path: |
+ reports/**
+ target/llvm-cov/**
+ coverage.lcov
+ codecov.json
+ if-no-files-found: ignore
+ retention-days: 7
diff --git a/.github/actions/cpp-bazel/pre-merge/action.yml b/.github/actions/cpp-bazel/pre-merge/action.yml
index eaa15c9c6..8b9ab0507 100644
--- a/.github/actions/cpp-bazel/pre-merge/action.yml
+++ b/.github/actions/cpp-bazel/pre-merge/action.yml
@@ -78,12 +78,16 @@ runs:
with:
tool: cargo-llvm-cov
+ - name: Download runtime binaries
+ if: inputs.task == 'test'
+ uses: ./.github/actions/utils/download-runtime
+
- name: Setup server for tests
id: iggy
if: inputs.task == 'test'
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
- name: Test with coverage
if: inputs.task == 'test'
diff --git a/.github/actions/csharp-dotnet/pre-merge/action.yml b/.github/actions/csharp-dotnet/pre-merge/action.yml
index d0471cd79..ea4a9a884 100644
--- a/.github/actions/csharp-dotnet/pre-merge/action.yml
+++ b/.github/actions/csharp-dotnet/pre-merge/action.yml
@@ -33,9 +33,11 @@ runs:
8.0.x
10.0.x
- - name: Setup Rust
+ - name: Download runtime binaries
if: startsWith(inputs.task, 'e2e')
- uses: ./.github/actions/utils/setup-rust
+ uses: ./.github/actions/utils/download-runtime
+ with:
+ install-system-dependencies: "false"
- name: Restore dependencies
run: |
@@ -81,7 +83,6 @@ runs:
if: startsWith(inputs.task, 'e2e')
shell: bash
run: |
- cargo build --locked --bin iggy-server --bin iggy
docker build \
-f core/server/Dockerfile \
--target runtime-prebuilt \
diff --git a/.github/actions/go/pre-merge/action.yml b/.github/actions/go/pre-merge/action.yml
index 30a4a0010..ebf640680 100644
--- a/.github/actions/go/pre-merge/action.yml
+++ b/.github/actions/go/pre-merge/action.yml
@@ -117,9 +117,9 @@ runs:
# Build the module
go build -v ./...
- - name: Setup Rust for e2e
+ - name: Download runtime binaries for e2e
if: inputs.task == 'e2e'
- uses: ./.github/actions/utils/setup-rust
+ uses: ./.github/actions/utils/download-runtime
# The Go SDK speaks the VSR wire protocol, so every e2e lane runs against
# a locally built VSR server rather than a container of the legacy one.
@@ -128,7 +128,7 @@ runs:
if: inputs.task == 'e2e'
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
replica-id: "0"
pid-file: ${{ runner.temp }}/iggy-go-e2e.pid
log-file: ${{ runner.temp }}/iggy-go-e2e.log
@@ -184,7 +184,7 @@ runs:
if: inputs.task == 'e2e'
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
replica-id: "0"
pid-file: ${{ runner.temp }}/iggy-go-e2e-tls.pid
log-file: ${{ runner.temp }}/iggy-go-e2e-tls.log
@@ -222,7 +222,7 @@ runs:
if: inputs.task == 'e2e'
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
replica-id: "0"
pid-file: ${{ runner.temp }}/iggy-go-cluster-0.pid
log-file: ${{ runner.temp }}/iggy-go-cluster-0.log
@@ -236,7 +236,7 @@ runs:
if: inputs.task == 'e2e'
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
replica-id: "1"
tcp_address: 127.0.0.1:8091
http_address: 127.0.0.1:3001
diff --git a/.github/actions/java-gradle/pre-merge/action.yml b/.github/actions/java-gradle/pre-merge/action.yml
index a6047a619..35920d0fc 100644
--- a/.github/actions/java-gradle/pre-merge/action.yml
+++ b/.github/actions/java-gradle/pre-merge/action.yml
@@ -76,16 +76,16 @@ runs:
# Exit with build exit code
exit $BUILD_EXIT_CODE
- - name: Setup Rust
+ - name: Download runtime binaries
if: inputs.task == 'test'
- uses: ./.github/actions/utils/setup-rust
+ uses: ./.github/actions/utils/download-runtime
- name: Start Iggy server
if: inputs.task == 'test'
id: iggy
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
wait-timeout-seconds: "90"
env:
IGGY_PATH: ${{ runner.temp }}/iggy-java-data
@@ -161,7 +161,7 @@ runs:
if: inputs.task == 'test'
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
wait-timeout-seconds: "90"
pid-file: ${{ runner.temp }}/iggy-server-tls.pid
log-file: ${{ runner.temp }}/iggy-server-tls.log
diff --git a/.github/actions/node-npm/pre-merge/action.yml b/.github/actions/node-npm/pre-merge/action.yml
index 4214810f1..97f0eb88d 100644
--- a/.github/actions/node-npm/pre-merge/action.yml
+++ b/.github/actions/node-npm/pre-merge/action.yml
@@ -37,9 +37,9 @@ runs:
run: npm install -g npm@11
shell: bash
- - name: Setup Rust
+ - name: Download runtime binaries
if: inputs.task == 'e2e'
- uses: ./.github/actions/utils/setup-rust
+ uses: ./.github/actions/utils/download-runtime
- name: Install netcat
if: inputs.task == 'e2e'
@@ -94,7 +94,7 @@ runs:
if: inputs.task == 'e2e'
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
wait-timeout-seconds: "90"
- name: E2E tests
@@ -125,7 +125,7 @@ runs:
if: inputs.task == 'e2e'
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
pid-file: ${{ runner.temp }}/iggy-server-tls.pid
log-file: ${{ runner.temp }}/iggy-server-tls.log
env:
diff --git a/.github/actions/php/pre-merge/action.yml b/.github/actions/php/pre-merge/action.yml
index 398c94234..8331338ea 100644
--- a/.github/actions/php/pre-merge/action.yml
+++ b/.github/actions/php/pre-merge/action.yml
@@ -153,12 +153,16 @@ runs:
run: |
"${PHP}" "${COMPOSER_BIN}" install --no-interaction --prefer-dist --no-progress
+ - name: Download runtime binaries
+ if: inputs.task == 'test'
+ uses: ./.github/actions/utils/download-runtime
+
- name: Start Iggy server
if: inputs.task == 'test'
id: iggy
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
- name: Run PHP SDK tests
if: inputs.task == 'test'
@@ -184,7 +188,7 @@ runs:
id: iggy-tls
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
pid-file: ${{ runner.temp }}/iggy-server-tls.pid
log-file: ${{ runner.temp }}/iggy-server-tls.log
env:
diff --git a/.github/actions/python-maturin/pre-merge/action.yml b/.github/actions/python-maturin/pre-merge/action.yml
index 3233c3a75..80ca9089f 100644
--- a/.github/actions/python-maturin/pre-merge/action.yml
+++ b/.github/actions/python-maturin/pre-merge/action.yml
@@ -28,14 +28,18 @@ runs:
steps:
- name: Setup Rust
uses: ./.github/actions/utils/setup-rust
+ with:
+ free-disk-space: ${{ inputs.task != 'test' }}
+
+ - name: Download runtime binaries
+ if: inputs.task == 'test'
+ uses: ./.github/actions/utils/download-runtime
- # setup-python changes PKG_CONFIG_PATH and invalidates cached native server dependencies.
- name: Build server Docker image for TLS tests
if: inputs.task == 'test'
run: |
# test_tls.py spawns the server in a container; build it from the
# same binaries the plain-TCP tests run against.
- cargo build --locked --bin iggy-server --bin iggy
docker build \
-f core/server/Dockerfile \
--target runtime-prebuilt \
@@ -49,7 +53,6 @@ runs:
id: iggy
uses: ./.github/actions/utils/server-start
with:
- mode: bin
bin: target/debug/iggy-server
- name: Setup Python
diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml
index 0180b9bf7..48bdbe86e 100644
--- a/.github/actions/rust/pre-merge/action.yml
+++ b/.github/actions/rust/pre-merge/action.yml
@@ -20,7 +20,7 @@ description: Rust pre-merge testing and linting github iggy actions
inputs:
task:
- description: "Task to run (check, check-msrv, fmt, clippy, clippy-macos-aarch64, sort, machete, doctest, verify-publish, test-build, test, test-storage-compat, miri)"
+ description: "Task to run (check, check-msrv, fmt, clippy, clippy-macos-aarch64, sort, machete, doctest, verify-publish, runtime, test-build, test, test-storage-compat, miri)"
required: true
component:
description: "Component name (for context)"
@@ -54,26 +54,26 @@ runs:
# Only test jobs execute nextest. Avoid downloading it in lint, metadata,
# and cross-build jobs.
install-nextest: ${{ inputs.task == 'test' || startsWith(inputs.task, 'test-') }}
- coverage: ${{ inputs.task == 'test-build' || inputs.task == 'test' }}
+ coverage: ${{ inputs.task == 'runtime' || inputs.task == 'test-build' || inputs.task == 'test' }}
# These tasks do not compile workspace crates and need no native
# libraries. All compiling tasks keep the existing dependency setup.
install-system-dependencies: ${{ inputs.task != 'fmt' && inputs.task != 'sort' && inputs.task != 'machete' }}
# Miri builds against a nightly toolchain with a separate `target/miri`
# subtree; isolate its cache from the stable `dev` namespace so the
# two don't evict each other.
- # check-msrv stays on `dev`: only `dev` is ever saved (by
- # coverage-baseline), so a dedicated key would be cold every run. It
+ # check-msrv stays on `dev`: only `dev` and `runtime` are ever saved
+ # (by coverage-baseline), so a dedicated key would be cold every run. It
# reads only (pre-merge save-cache=false), so it cannot churn `dev`;
# the warm `~/.cargo` restore is a free win, and its 1.95 target
# artifacts rebuild regardless since the rustc hash differs.
- shared-key: ${{ inputs.task == 'miri' && 'miri' || 'dev' }}
+ shared-key: ${{ inputs.task == 'miri' && 'miri' || inputs.task == 'runtime' && 'runtime' || 'dev' }}
# Shards download the build artifact. Metadata-only tasks need no target cache.
read-cache: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'machete' || inputs.task == 'test') && 'false' || 'true' }}
save-cache: ${{ inputs.save-cache }}
# Light legs fit in the runner's ~89 GiB default headroom; skip the
# ~20-45s reclaim. Disk-heavy legs (coverage build + testcontainers
# images on test jobs, cross-builds, miri, verify-publish) keep it.
- free-disk-space: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'check-msrv' || inputs.task == 'machete' || inputs.task == 'doctest') && 'false' || 'true' }}
+ free-disk-space: ${{ (inputs.task == 'runtime' || inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'check-msrv' || inputs.task == 'machete' || inputs.task == 'doctest') && 'false' || 'true' }}
free-disk-space-aggressive: ${{ inputs.free-disk-space-aggressive }}
- name: Install cargo-sort
@@ -174,69 +174,81 @@ runs:
rm -f "$HOME"/.local/share/keyrings/*
shell: bash
- - name: Build coverage test archive
- if: inputs.task == 'test-build'
+ - name: Download runtime binaries
+ if: inputs.task == 'test' || inputs.task == 'test-storage-compat'
+ uses: ./.github/actions/utils/download-runtime
+ with:
+ restore: "false"
+
+ - name: Build coverage artifacts
+ if: inputs.task == 'runtime' || inputs.task == 'test-build'
env:
PLAN_FILE: ${{ steps.rail.outputs.plan-file }}
PLAN_STATUS: ${{ steps.rail.outcome }}
TEST_SELECTION: ${{ inputs.test-scope }}
+ ARTIFACT_TASK: ${{ inputs.task }}
run: |
set -euo pipefail
- ARTIFACT_DIR="$RUNNER_TEMP/rust-test-build"
+ ARTIFACT_DIR="$RUNNER_TEMP/rust-$ARTIFACT_TASK"
mkdir -p "$ARTIFACT_DIR"
- # Read only validated selectors. Argument and package selectors are
- # NUL-delimited, hence `mapfile -d ''`. Any planner or selector failure
- # restores the full-workspace build and test.
- # target-args emits `--test` pairs only when every selected target of
- # every selected package is an integration test, so it never drops a
- # package or its unit tests from a mixed selection.
- SELECTOR_DIR="${RUNNER_TEMP:?RUNNER_TEMP is required}/cargo-rail-selectors"
+ git rev-parse HEAD > "$ARTIFACT_DIR/revision"
+ if [[ "$ARTIFACT_TASK" == runtime ]]; then
+ # Workspace target selection keeps the dependency feature graph stable across SDK PRs.
+ cargo build --locked --bin iggy-server --bin iggy
+ tar --zstd -cf "$ARTIFACT_DIR/runtime.tar.zst" target/debug/iggy-server target/debug/iggy
+ exit 0
+ fi
+
+ SELECTOR_DIR="$RUNNER_TEMP/cargo-rail-selectors"
mkdir -p "$SELECTOR_DIR"
TEST_CARGO_ARGS=()
TEST_TARGET_ARGS=()
TEST_PACKAGES=()
- if [[ "$TEST_SELECTION" == "workspace" ]]; then
+ RUNTIME_PACKAGES=()
+ if [[ "$TEST_SELECTION" == workspace ]]; then
TEST_SCOPE=workspace
RUNTIME_SCOPE=workspace
- elif [[ "$TEST_SELECTION" != "affected" ]]; then
+ elif [[ "$TEST_SELECTION" != affected ]]; then
echo "::error::Invalid test scope: $TEST_SELECTION"
exit 1
- elif [[ "$PLAN_STATUS" == "success" ]] &&
+ elif [[ "$PLAN_STATUS" == success ]] &&
TEST_SCOPE=$(cargo-rail-action plan cargo-scope "$PLAN_FILE" cargo.test) &&
RUNTIME_SCOPE=$(cargo-rail-action plan cargo-scope "$PLAN_FILE" runtime-artifacts) &&
cargo-rail-action plan cargo-args "$PLAN_FILE" cargo.test > "$SELECTOR_DIR/test-args" &&
cargo-rail-action plan target-args "$PLAN_FILE" cargo.test > "$SELECTOR_DIR/test-target-args" &&
cargo-rail-action plan package-names "$PLAN_FILE" cargo.test > "$SELECTOR_DIR/test-packages" &&
cargo-rail-action plan package-names "$PLAN_FILE" runtime-artifacts > "$SELECTOR_DIR/runtime-packages"; then
+ # NUL-delimited selectors preserve package and argument boundaries.
mapfile -d '' -t TEST_CARGO_ARGS < "$SELECTOR_DIR/test-args"
mapfile -d '' -t TEST_TARGET_ARGS < "$SELECTOR_DIR/test-target-args"
mapfile -d '' -t TEST_PACKAGES < "$SELECTOR_DIR/test-packages"
+ mapfile -d '' -t RUNTIME_PACKAGES < "$SELECTOR_DIR/runtime-packages"
else
TEST_SCOPE=workspace
RUNTIME_SCOPE=workspace
echo "::warning::Cargo-Rail plan unavailable or invalid; running the full workspace"
fi
-
printf '%s\n' "$TEST_SCOPE" > "$ARTIFACT_DIR/scope"
- git rev-parse HEAD > "$ARTIFACT_DIR/revision"
- if [[ "$TEST_SCOPE" == "skipped" ]]; then
+ if [[ "$TEST_SCOPE" == skipped ]]; then
echo "::notice::Cargo-Rail reports no affected Rust tests"
exit 0
fi
- # Only spawned binaries and dlopened plugins need a separate dev build.
- # nextest archive compiles the selected tests with their dev-dependency features.
+ # SDKs need only server/CLI. Keep Rust-only binaries and plugins with their tests.
BUILD_ARGS=()
- if [[ "$RUNTIME_SCOPE" == "packages" ]]; then
- mapfile -d '' -t RUNTIME_PACKAGES < "$SELECTOR_DIR/runtime-packages"
+ if [[ "$RUNTIME_SCOPE" == workspace ]]; then
+ BUILD_ARGS=(--workspace --exclude server --exclude iggy-cli)
+ elif [[ "$RUNTIME_SCOPE" == packages ]]; then
for package in "${RUNTIME_PACKAGES[@]}"; do
- BUILD_ARGS+=(-p "$package")
+ if [[ "$package" != server && "$package" != iggy-cli ]]; then
+ BUILD_ARGS+=(-p "$package")
+ fi
done
fi
- : > "$ARTIFACT_DIR/build.jsonl"
+
: > "$ARTIFACT_DIR/runtime-files"
- if [[ "$RUNTIME_SCOPE" != "skipped" ]]; then
+ if (( ${#BUILD_ARGS[@]} )); then
cargo build --locked "${BUILD_ARGS[@]}" --message-format=json-render-diagnostics > "$ARTIFACT_DIR/build.jsonl"
jq -r 'select(.reason == "compiler-artifact") |
(select(.target.kind | index("bin")) | .executable // empty),
@@ -245,9 +257,10 @@ runs:
while IFS= read -r artifact; do
realpath --relative-to="$GITHUB_WORKSPACE" "$artifact"
done > "$ARTIFACT_DIR/runtime-files"
+ rm "$ARTIFACT_DIR/build.jsonl"
fi
tar --zstd -cf "$ARTIFACT_DIR/runtime.tar.zst" -T "$ARTIFACT_DIR/runtime-files"
- rm "$ARTIFACT_DIR/build.jsonl" "$ARTIFACT_DIR/runtime-files"
+ rm "$ARTIFACT_DIR/runtime-files"
# api_handler_tests, version_firewall_tests, and server_e2e_tests need gitignored
# wire fixtures. Master always generates them. On PRs, use the selected
@@ -336,7 +349,6 @@ runs:
echo "::notice::Cargo-Rail reports no affected Rust tests"
exit 0
fi
- tar --zstd -xf "$ARTIFACT_DIR/runtime.tar.zst"
if [[ "$(cat "$ARTIFACT_DIR/kafka-fixtures")" == true ]]; then
tar --zstd -xf "$ARTIFACT_DIR/kafka-fixtures.tar.zst"
# Fixtures are expected to exist from here on; a missing one now means
@@ -385,21 +397,23 @@ runs:
# BDD targets require features, so a BDD-only scope can contain no runnable tests.
NO_TESTS=warn
fi
- cargo nextest run --archive-file "$ARTIFACT_DIR/nextest.tar.zst" \
- --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --workspace-remap "$GITHUB_WORKSPACE" \
+ # nextest can archive non-test binaries with dev-dependency features.
+ # Restore the shared runtime last so every suite executes the same binaries.
+ tar --zstd -xf "$ARTIFACT_DIR/nextest.tar.zst"
+ tar --zstd -xf "$ARTIFACT_DIR/runtime.tar.zst"
+ ./scripts/ci/restore-runtime.sh "$RUNNER_TEMP/rust-runtime"
+ cargo nextest run \
+ --cargo-metadata target/nextest/cargo-metadata.json \
+ --binaries-metadata target/nextest/binaries-metadata.json \
+ --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
--no-fail-fast --no-tests="$NO_TESTS" --profile ci \
--partition "hash:${PARTITION_INDEX}/${PARTITION_COUNT}"
shell: bash
- # On-disk format backwards compatibility. The script builds an iggy-server
- # from the master tip under the checked-out merge ref (origin/master off a
- # pull_request run) and one from HEAD, seeds a data directory with the
- # former and reads it back with the latter. It does its own shallow fetch
- # of the baseline commit and exits non-zero if it cannot resolve one, so
- # the default depth-1 checkout is sufficient here.
+ # Compare the shared HEAD runtime with a separately built baseline server.
- name: Storage format backwards compatibility
if: inputs.task == 'test-storage-compat'
- run: ./scripts/ci/storage-compat.sh
+ run: ./scripts/ci/storage-compat.sh --runtime-artifacts "$RUNNER_TEMP/rust-runtime"
shell: bash
# Miri (UB detector) on the unsafe-heavy crates that don't pull tokio /
diff --git a/.github/actions/swift/pre-merge/action.yml b/.github/actions/swift/pre-merge/action.yml
index 29591c227..077f49ef3 100644
--- a/.github/actions/swift/pre-merge/action.yml
+++ b/.github/actions/swift/pre-merge/action.yml
@@ -105,10 +105,6 @@ runs:
fi
echo "✅ golden.json matches the Rust crates"
- - name: Setup Rust for e2e
- if: inputs.task == 'e2e'
- uses: ./.github/actions/utils/setup-rust
-
- name: Build the end-to-end suite
shell: bash
if: inputs.task == 'e2e'
@@ -116,14 +112,18 @@ runs:
cd foreign/swift
swift build --build-tests
- # The SDK speaks the VSR wire protocol, so every e2e lane runs against a
- # locally built VSR server.
+ - name: Download runtime binaries
+ if: inputs.task == 'e2e'
+ uses: ./.github/actions/utils/download-runtime
+
+ # The SDK speaks the VSR wire protocol, so every e2e lane runs against the
+ # VSR server from the shared runtime archive.
- name: Start Iggy server (plain)
id: iggy
if: inputs.task == 'e2e'
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
replica-id: "0"
pid-file: ${{ runner.temp }}/iggy-swift-e2e.pid
log-file: ${{ runner.temp }}/iggy-swift-e2e.log
@@ -155,7 +155,7 @@ runs:
if: inputs.task == 'e2e'
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
replica-id: "0"
pid-file: ${{ runner.temp }}/iggy-swift-e2e-tls.pid
log-file: ${{ runner.temp }}/iggy-swift-e2e-tls.log
@@ -196,7 +196,7 @@ runs:
if: inputs.task == 'e2e'
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
replica-id: "0"
pid-file: ${{ runner.temp }}/iggy-swift-cluster-0.pid
log-file: ${{ runner.temp }}/iggy-swift-cluster-0.log
@@ -210,7 +210,7 @@ runs:
if: inputs.task == 'e2e'
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
replica-id: "1"
tcp_address: 127.0.0.1:8091
http_address: 127.0.0.1:3001
diff --git a/.github/actions/utils/download-runtime/action.yml b/.github/actions/utils/download-runtime/action.yml
new file mode 100644
index 000000000..a0e18b567
--- /dev/null
+++ b/.github/actions/utils/download-runtime/action.yml
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: download-runtime
+description: >-
+ Restore coverage-instrumented runtime binaries for this workflow revision.
+ In .github/workflows/_detect.yml, needsRuntime must select the caller's
+ task, and in .github/workflows/pre-merge.yml the caller's job must need
+ runtime.
+
+inputs:
+ restore:
+ description: Extract and verify the archive after downloading
+ default: "true"
+ install-system-dependencies:
+ description: Install libraries for running the binaries on the host
+ default: "true"
+
+runs:
+ using: composite
+ steps:
+ - name: Install runtime libraries
+ if: inputs.install-system-dependencies == 'true'
+ shell: bash
+ run: |
+ missing=()
+ for package in libhwloc-dev libudev-dev; do
+ if [[ "$(dpkg-query -W -f='${Status}' "$package" 2>/dev/null)" != 'install ok installed' ]]; then
+ missing+=("$package")
+ fi
+ done
+ if (( ${#missing[@]} )); then
+ ./scripts/ci/apt-install.sh "${missing[@]}"
+ fi
+
+ - name: Download runtime archive
+ uses: actions/download-artifact@v8
+ with:
+ name: rust-runtime-${{ github.run_id }}
+ path: ${{ runner.temp }}/rust-runtime
+
+ - name: Verify and restore runtime binaries
+ if: inputs.restore == 'true'
+ shell: bash
+ run: ./scripts/ci/restore-runtime.sh "$RUNNER_TEMP/rust-runtime"
diff --git a/.github/actions/utils/server-start/action.yml b/.github/actions/utils/server-start/action.yml
index 6b76e9cc5..31e6bfe1f 100644
--- a/.github/actions/utils/server-start/action.yml
+++ b/.github/actions/utils/server-start/action.yml
@@ -18,24 +18,11 @@
name: server-start
description: Start Iggy server and wait for readiness
inputs:
- mode:
- description: "How to run the server: cargo|bin"
- required: false
- default: "cargo"
- cargo-bin:
- description: "Cargo binary name (when mode=cargo)"
- required: false
- default: "iggy-server"
- cargo-profile:
- description: "Cargo profile: release|debug"
- required: false
- default: "debug"
bin:
- description: "Path to server binary (when mode=bin)"
- required: false
- default: ""
+ description: "Path to server binary"
+ required: true
working-directory:
- description: "Working directory for building/running"
+ description: "Working directory for running the server"
required: false
default: "."
tcp_address:
@@ -97,35 +84,6 @@ runs:
echo "LOG_FILE=$LOG_FILE" >> "$GITHUB_ENV"
echo "PID_FILE=$PID_FILE" >> "$GITHUB_ENV"
- - id: resolve-bin
- shell: bash
- working-directory: ${{ inputs.working-directory }}
- run: |
- set -euo pipefail
- MODE="${{ inputs.mode }}"
- BIN_PATH=""
- if [[ "$MODE" == "cargo" ]]; then
- PROFILE="${{ inputs.cargo-profile }}"
- NAME="${{ inputs.cargo-bin }}"
- if [[ "$PROFILE" == "release" ]]; then
- OUT="target/release/$NAME"
- else
- OUT="target/debug/$NAME"
- fi
- # An existing executable may belong to a different checkout or feature set.
- echo "Building $NAME with cargo ($PROFILE)..."
- CARGO_ARGS=(build --locked --bin "$NAME")
- if [[ "$PROFILE" == "release" ]]; then
- CARGO_ARGS+=(--release)
- fi
- cargo "${CARGO_ARGS[@]}"
- BIN_PATH="$OUT"
- else
- BIN_PATH="${{ inputs.bin }}"
- [[ -x "$BIN_PATH" ]] || { echo "Binary not found or not executable: $BIN_PATH"; exit 1; }
- fi
- echo "bin=$BIN_PATH" >> "$GITHUB_OUTPUT"
-
- id: busy
shell: bash
run: |
@@ -152,6 +110,8 @@ runs:
working-directory: ${{ inputs.working-directory }}
run: |
set -euo pipefail
+ [[ -n "${{ inputs.bin }}" ]] || { echo "::error::Input required and not supplied: bin"; exit 1; }
+ [[ -x "${{ inputs.bin }}" ]] || { echo "::error::Binary not found or not executable: ${{ inputs.bin }}"; exit 1; }
export IGGY_TCP_ADDRESS="${{ inputs.tcp_address }}"
export IGGY_HTTP_ADDRESS="${{ inputs.http_address }}"
export IGGY_ROOT_USERNAME="iggy"
@@ -160,7 +120,8 @@ runs:
if [[ -n "${{ inputs.replica-id }}" ]]; then
CLI_ARGS+=(--replica-id "${{ inputs.replica-id }}")
fi
- nohup "${{ steps.resolve-bin.outputs.bin }}" "${CLI_ARGS[@]}" >"$LOG_FILE" 2>&1 &
+ export LLVM_PROFILE_FILE="$RUNNER_TEMP/iggy-server-%h-%p-%m.profraw"
+ nohup "${{ inputs.bin }}" "${CLI_ARGS[@]}" >"$LOG_FILE" 2>&1 &
echo $! > "$PID_FILE"
echo "Started server PID $(cat "$PID_FILE") → logs: $LOG_FILE"
echo "Server configured with TCP=$IGGY_TCP_ADDRESS, HTTP=$IGGY_HTTP_ADDRESS"
diff --git a/.github/config/components.yml b/.github/config/components.yml
index fe40f8f48..c0be31c6d 100644
--- a/.github/config/components.yml
+++ b/.github/config/components.yml
@@ -65,6 +65,7 @@ components:
ci-infrastructure:
paths:
- ".github/**"
+ - "scripts/ci/restore-runtime.sh"
# Core library components that others depend on
rust-sdk:
@@ -564,6 +565,9 @@ components:
# CI/CD workflow monitoring (informational, no tasks)
ci-workflows:
paths:
+ - ".github/config/components.yml"
+ - "scripts/ci/restore-runtime.sh"
+ - "scripts/ci/test-runtime-artifacts.py"
- ".github/workflows/**/*.yml"
- ".github/actions/**/*.yml"
- ".github/ci/**/*.yml"
diff --git a/.github/workflows/_build_ci_artifacts.yml b/.github/workflows/_build_ci_artifacts.yml
new file mode 100644
index 000000000..5ffcabd9c
--- /dev/null
+++ b/.github/workflows/_build_ci_artifacts.yml
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: _build_ci_artifacts
+on:
+ workflow_call:
+ inputs:
+ task:
+ description: "Artifact producer: runtime or test-build"
+ type: string
+ required: true
+ test-scope:
+ type: string
+ default: affected
+ save-cache:
+ type: boolean
+ default: false
+ build-metadata:
+ type: boolean
+ default: false
+ timeout-minutes:
+ type: number
+ default: 120
+
+permissions:
+ contents: read
+
+env:
+ IGGY_CI_BUILD: ${{ inputs.build-metadata }}
+
+jobs:
+ build:
+ name: build
+ runs-on: ubuntu-26.04
+ timeout-minutes: ${{ inputs.timeout-minutes }}
+ steps:
+ - uses: actions/checkout@v7.0.1
+
+ - name: Prepare artifacts
+ uses: ./.github/actions/rust/pre-merge
+ with:
+ task: ${{ inputs.task }}
+ test-scope: ${{ inputs.test-scope }}
+ save-cache: ${{ inputs.save-cache }}
+ free-disk-space-aggressive: ${{ inputs.test-scope == 'workspace' }}
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v7
+ with:
+ name: rust-${{ inputs.task }}-${{ github.run_id }}
+ path: ${{ runner.temp }}/rust-${{ inputs.task }}/
+ compression-level: 0
+ if-no-files-found: error
+ overwrite: true
+ retention-days: 30
diff --git a/.github/workflows/_detect.yml b/.github/workflows/_detect.yml
index 8cd417c1c..fe8ec7e68 100644
--- a/.github/workflows/_detect.yml
+++ b/.github/workflows/_detect.yml
@@ -20,12 +20,18 @@ name: _detect
on:
workflow_call:
outputs:
+ runtime:
+ description: "Whether selected tests need shared runtime binaries"
+ value: ${{ jobs.detect.outputs.runtime }}
third_party_licenses:
description: "Whether third-party license inputs changed"
value: ${{ jobs.detect.outputs.third_party_licenses }}
rust_plan:
- description: "Selected Rust checks and tests; empty when Rust is unaffected"
+ description: "Selected Rust checks that need no CI artifacts; empty when none run"
value: ${{ jobs.detect.outputs.rust_plan }}
+ rust_runtime_plan:
+ description: "Selected Rust tests and storage-compat checks, which need CI artifacts; empty when none run"
+ value: ${{ jobs.detect.outputs.rust_runtime_plan }}
python_matrix:
description: "Matrix for Python SDK"
value: ${{ jobs.detect.outputs.python_matrix }}
@@ -68,8 +74,10 @@ jobs:
runs-on: ubuntu-slim
timeout-minutes: 10
outputs:
+ runtime: ${{ steps.mk.outputs.runtime }}
third_party_licenses: ${{ github.event_name != 'pull_request' || steps.changed.outputs.third_party_licenses == 'true' }}
rust_plan: ${{ steps.mk.outputs.rust_plan }}
+ rust_runtime_plan: ${{ steps.mk.outputs.rust_runtime_plan }}
python_matrix: ${{ steps.mk.outputs.python_matrix }}
php_matrix: ${{ steps.mk.outputs.php_matrix }}
node_matrix: ${{ steps.mk.outputs.node_matrix }}
@@ -426,9 +434,22 @@ jobs:
require('fs').appendFileSync(process.env.GITHUB_OUTPUT, `${output}\n`);
};
- setOutput('rust_plan', groups.rust.length ? JSON.stringify({
- checks: groups.rust.filter(entry => entry.task !== 'test'),
- tests: groups.rust.some(entry => entry.task === 'test'),
+ const rustRuntime = groups.rust.filter(({task}) => task === 'test' || task === 'test-storage-compat');
+ const rustChecks = groups.rust.filter(entry => !rustRuntime.includes(entry));
+ const needsRuntime = rustRuntime.length > 0 ||
+ groups.bdd.length > 0 || groups.examples.length > 0 ||
+ ['python', 'php', 'java', 'cpp'].some(group => groups[group].some(({task}) => task === 'test')) ||
+ ['node', 'go', 'csharp', 'swift'].some(group => groups[group].some(({task}) => task.startsWith('e2e')));
+ setOutput('runtime', String(needsRuntime));
+ for (const group of ['bdd', 'examples']) {
+ groups[group] = groups[group].map(entry => ({...entry, name: entry.task.slice(group.length + 1)}));
+ }
+ groups.other = groups.other.map(entry => ({...entry, name: `${entry.component}/${entry.task}`}));
+
+ setOutput('rust_plan', rustChecks.length ? JSON.stringify({checks: rustChecks, tests: false}) : '');
+ setOutput('rust_runtime_plan', rustRuntime.length ? JSON.stringify({
+ checks: rustRuntime.filter(entry => entry.task !== 'test'),
+ tests: rustRuntime.some(entry => entry.task === 'test'),
}) : '');
setOutput('python_matrix', JSON.stringify(matrix(groups.python)));
setOutput('php_matrix', JSON.stringify(matrix(groups.php)));
diff --git a/.github/workflows/_rust.yml b/.github/workflows/_rust.yml
index 7d5458b72..e826e5c54 100644
--- a/.github/workflows/_rust.yml
+++ b/.github/workflows/_rust.yml
@@ -23,21 +23,6 @@ on:
description: "JSON with checks (component/task entries) and tests (boolean)"
type: string
required: true
- test-scope:
- description: "Test selection: affected or workspace"
- type: string
- default: affected
- timeout-minutes:
- type: number
- default: 60
- save-cache:
- description: "Save dependencies from the test build for subsequent PRs"
- type: boolean
- default: false
- build-metadata:
- description: "Embed CI build metadata in the server binaries"
- type: boolean
- default: false
secrets:
CODECOV_TOKEN:
required: false
@@ -46,12 +31,9 @@ permissions:
contents: read
issues: read # live breaking:storage label lookup
-env:
- IGGY_CI_BUILD: ${{ inputs.build-metadata }}
-
jobs:
checks:
- name: ${{ matrix.component }}/${{ matrix.task }}
+ name: ${{ matrix.component == 'rust' && matrix.task || format('{0}/{1}', matrix.component, matrix.task) }}
if: ${{ fromJSON(inputs.plan).checks[0] != null }}
strategy:
fail-fast: true
@@ -63,7 +45,7 @@ jobs:
matrix.task == 'build-windows-sdk' && 'windows-latest' ||
'ubuntu-26.04'
}}
- timeout-minutes: ${{ inputs.timeout-minutes }}
+ timeout-minutes: 60
steps:
- uses: actions/checkout@v7.0.1
@@ -109,88 +91,9 @@ jobs:
if-no-files-found: ignore
retention-days: 7
- test-build:
- name: Build test artifacts
- if: ${{ fromJSON(inputs.plan).tests }}
- runs-on: ubuntu-26.04
- timeout-minutes: ${{ inputs.timeout-minutes }}
- steps:
- - uses: actions/checkout@v7.0.1
-
- - name: Build coverage test archive
- uses: ./.github/actions/rust/pre-merge
- with:
- task: test-build
- test-scope: ${{ inputs.test-scope }}
- save-cache: ${{ inputs.save-cache }}
- free-disk-space-aggressive: ${{ inputs.test-scope == 'workspace' }}
-
- - name: Upload test artifacts
- uses: actions/upload-artifact@v7
- with:
- name: rust-test-build-${{ github.run_id }}
- path: ${{ runner.temp }}/rust-test-build/
- compression-level: 0
- if-no-files-found: error
- overwrite: true
- retention-days: 7
-
tests:
- name: Test shard ${{ matrix.partition }}/${{ strategy.job-total }}
- needs: test-build
- strategy:
- fail-fast: false
- matrix:
- partition: [1, 2, 3]
- runs-on: ubuntu-26.04
- timeout-minutes: ${{ inputs.timeout-minutes }}
- steps:
- - uses: actions/checkout@v7.0.1
-
- - name: Download test artifacts
- uses: actions/download-artifact@v8
- with:
- name: rust-test-build-${{ github.run_id }}
- path: ${{ runner.temp }}/rust-test-build
-
- - name: Run test shard
- uses: ./.github/actions/rust/pre-merge
- with:
- task: test
- partition: ${{ matrix.partition }}
- partitions: ${{ strategy.job-total }}
-
- - name: Generate coverage report
- if: always() && !cancelled()
- run: |
- if compgen -G 'target/*.profraw' > /dev/null; then
- cargo llvm-cov report --codecov --output-path codecov.json
- else
- echo "::notice::No coverage profiles found; skipping report"
- fi
- shell: bash
-
- - name: Upload coverage to Codecov
- if: always() && !cancelled() && hashFiles('codecov.json') != ''
- uses: codecov/codecov-action@v7.1.1
- with:
- token: ${{ secrets.CODECOV_TOKEN }}
- files: codecov.json
- disable_search: true
- flags: rust
- fail_ci_if_error: false
- verbose: true
- override_pr: ${{ github.event.pull_request.number }}
-
- - name: Upload reports
- if: always()
- uses: actions/upload-artifact@v7
- with:
- name: rust-test-${{ matrix.partition }}-reports-${{ github.run_id }}-${{ github.run_attempt }}
- path: |
- reports/**
- target/llvm-cov/**
- coverage.lcov
- codecov.json
- if-no-files-found: ignore
- retention-days: 7
+ name: tests
+ if: ${{ fromJSON(inputs.plan).tests }}
+ uses: ./.github/workflows/_rust_tests.yml
+ secrets:
+ CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.github/workflows/_rust_tests.yml b/.github/workflows/_rust_tests.yml
new file mode 100644
index 000000000..870c2b0cd
--- /dev/null
+++ b/.github/workflows/_rust_tests.yml
@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: _rust_tests
+on:
+ workflow_call:
+ inputs:
+ timeout-minutes:
+ type: number
+ default: 60
+ secrets:
+ CODECOV_TOKEN:
+ required: false
+
+permissions:
+ contents: read
+
+jobs:
+ tests:
+ name: test (${{ matrix.partition }}/${{ strategy.job-total }})
+ strategy:
+ fail-fast: false
+ matrix:
+ partition: [1, 2, 3]
+ runs-on: ubuntu-26.04
+ timeout-minutes: ${{ inputs.timeout-minutes }}
+ steps:
+ - uses: actions/checkout@v7.0.1
+
+ - name: Download test artifacts
+ uses: actions/download-artifact@v8
+ with:
+ name: rust-test-build-${{ github.run_id }}
+ path: ${{ runner.temp }}/rust-test-build
+
+ - name: Run test shard
+ uses: ./.github/actions/rust/pre-merge
+ with:
+ task: test
+ partition: ${{ matrix.partition }}
+ partitions: ${{ strategy.job-total }}
+
+ - name: Generate coverage report
+ if: always() && !cancelled()
+ run: |
+ if compgen -G 'target/*.profraw' > /dev/null; then
+ cargo llvm-cov report --codecov --output-path codecov.json
+ else
+ echo "::notice::No coverage profiles found; skipping report"
+ fi
+ shell: bash
+
+ - name: Upload coverage to Codecov
+ if: always() && !cancelled() && hashFiles('codecov.json') != ''
+ uses: codecov/codecov-action@v7.1.1
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
+ files: codecov.json
+ disable_search: true
+ flags: rust
+ fail_ci_if_error: false
+ verbose: true
+ override_pr: ${{ github.event.pull_request.number }}
+
+ - name: Upload reports
+ if: always()
+ uses: actions/upload-artifact@v7
+ with:
+ name: rust-test-${{ matrix.partition }}-reports-${{ github.run_id }}-${{ github.run_attempt }}
+ path: |
+ reports/**
+ target/llvm-cov/**
+ coverage.lcov
+ codecov.json
+ if-no-files-found: ignore
+ retention-days: 7
diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml
index 0d24d4da2..90323f2ab 100644
--- a/.github/workflows/_test.yml
+++ b/.github/workflows/_test.yml
@@ -19,14 +19,13 @@ name: _test
on:
workflow_call:
inputs:
- component:
+ matrix:
type: string
required: true
- description: "Component to test"
- task:
- type: string
- required: true
- description: "Task to run"
+ description: "Selected component/task entries"
+ fail-fast:
+ type: boolean
+ default: true
secrets:
CODECOV_TOKEN:
required: false
@@ -38,9 +37,13 @@ permissions:
jobs:
run:
+ name: ${{ matrix.name || matrix.task }}
+ strategy:
+ fail-fast: ${{ inputs.fail-fast }}
+ matrix: ${{ fromJSON(inputs.matrix) }}
# Select runner based on build target
runs-on: ${{
- inputs.task == 'build-macos' && 'macos-15' ||
+ matrix.task == 'build-macos' && 'macos-15' ||
'ubuntu-26.04'
}}
timeout-minutes: 60
@@ -48,23 +51,19 @@ jobs:
- name: Checkout code
uses: actions/checkout@v7.0.1
- - name: Skip noop
- if: inputs.component == 'noop'
- run: echo "No changes detected, skipping tests"
-
# Python SDK
- name: Set up Docker Buildx for Python
- if: inputs.component == 'sdk-python' && inputs.task == 'test'
+ if: matrix.component == 'sdk-python' && matrix.task == 'test'
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
- name: Run Python SDK task
- if: inputs.component == 'sdk-python'
+ if: matrix.component == 'sdk-python'
uses: ./.github/actions/python-maturin/pre-merge
with:
- task: ${{ inputs.task }}
+ task: ${{ matrix.task }}
- name: Upload Python coverage to Codecov
- if: inputs.component == 'sdk-python' && inputs.task == 'test'
+ if: matrix.component == 'sdk-python' && matrix.task == 'test'
uses: codecov/codecov-action@v7.1.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
@@ -77,13 +76,13 @@ jobs:
# PHP SDK
- name: Run PHP SDK task
- if: inputs.component == 'sdk-php'
+ if: matrix.component == 'sdk-php'
uses: ./.github/actions/php/pre-merge
with:
- task: ${{ inputs.task }}
+ task: ${{ matrix.task }}
- name: Upload PHP coverage to Codecov
- if: inputs.component == 'sdk-php' && inputs.task == 'test'
+ if: matrix.component == 'sdk-php' && matrix.task == 'test'
uses: codecov/codecov-action@v7.1.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
@@ -96,44 +95,44 @@ jobs:
# Node SDK
- name: Run Node SDK task
- if: inputs.component == 'sdk-node'
+ if: matrix.component == 'sdk-node'
uses: ./.github/actions/node-npm/pre-merge
with:
- task: ${{ inputs.task }}
+ task: ${{ matrix.task }}
- name: Upload Node coverage to Codecov
if: >-
- inputs.component == 'sdk-node' &&
- (inputs.task == 'test' ||
- inputs.task == 'e2e')
+ matrix.component == 'sdk-node' &&
+ (matrix.task == 'test' ||
+ matrix.task == 'e2e')
uses: codecov/codecov-action@v7.1.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: >-
reports/node-coverage/${{
- inputs.task == 'test' && 'unit' || inputs.task
+ matrix.task == 'test' && 'unit' || matrix.task
}}/lcov.info
disable_search: true
flags: >-
- ${{ inputs.task == 'test' && 'node' ||
- format('node-{0}', inputs.task) }}
+ ${{ matrix.task == 'test' && 'node' ||
+ format('node-{0}', matrix.task) }}
fail_ci_if_error: false
verbose: true
override_pr: ${{ github.event.pull_request.number }}
# Go SDK
- name: Run Go SDK task
- if: inputs.component == 'sdk-go'
+ if: matrix.component == 'sdk-go'
uses: ./.github/actions/go/pre-merge
with:
- task: ${{ inputs.task }}
+ task: ${{ matrix.task }}
- name: Upload Go coverage to Codecov
- if: inputs.component == 'sdk-go' && (inputs.task == 'test' || inputs.task == 'e2e')
+ if: matrix.component == 'sdk-go' && (matrix.task == 'test' || matrix.task == 'e2e')
uses: codecov/codecov-action@v7.1.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
- files: reports/${{ inputs.task == 'test' && 'go-coverage.out' || 'go-coverage-e2e.out' }}
+ files: reports/${{ matrix.task == 'test' && 'go-coverage.out' || 'go-coverage-e2e.out' }}
disable_search: true
flags: go
fail_ci_if_error: false
@@ -142,34 +141,34 @@ jobs:
# Java SDK
- name: Run Java SDK task
- if: inputs.component == 'sdk-java'
+ if: matrix.component == 'sdk-java'
uses: ./.github/actions/java-gradle/pre-merge
with:
- task: ${{ inputs.task }}
+ task: ${{ matrix.task }}
# C# SDK
- name: Run C# SDK task
- if: inputs.component == 'sdk-csharp'
+ if: matrix.component == 'sdk-csharp'
uses: ./.github/actions/csharp-dotnet/pre-merge
with:
- task: ${{ inputs.task }}
+ task: ${{ matrix.task }}
# C++ SDK
- name: Run C++ SDK Task
- if: inputs.component == 'sdk-cpp'
+ if: matrix.component == 'sdk-cpp'
uses: ./.github/actions/cpp-bazel/pre-merge
with:
- task: ${{ inputs.task }}
+ task: ${{ matrix.task }}
# Swift SDK
- name: Run Swift SDK task
- if: inputs.component == 'sdk-swift'
+ if: matrix.component == 'sdk-swift'
uses: ./.github/actions/swift/pre-merge
with:
- task: ${{ inputs.task }}
+ task: ${{ matrix.task }}
- name: Upload C++ coverage to Codecov
- if: inputs.component == 'sdk-cpp' && inputs.task == 'test'
+ if: matrix.component == 'sdk-cpp' && matrix.task == 'test'
uses: codecov/codecov-action@v7.1.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
@@ -181,7 +180,7 @@ jobs:
override_pr: ${{ github.event.pull_request.number }}
- name: Upload Java coverage to Codecov
- if: inputs.component == 'sdk-java' && inputs.task == 'test'
+ if: matrix.component == 'sdk-java' && matrix.task == 'test'
uses: codecov/codecov-action@v7.1.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
@@ -193,7 +192,7 @@ jobs:
override_pr: ${{ github.event.pull_request.number }}
- name: Upload C# coverage to Codecov
- if: inputs.component == 'sdk-csharp' && (inputs.task == 'test' || startsWith(inputs.task, 'e2e'))
+ if: matrix.component == 'sdk-csharp' && (matrix.task == 'test' || startsWith(matrix.task, 'e2e'))
uses: codecov/codecov-action@v7.1.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
@@ -206,54 +205,54 @@ jobs:
# Web UI
- name: Setup Node.js for Web UI
- if: inputs.component == 'web-ui'
+ if: matrix.component == 'web-ui'
uses: actions/setup-node@v7
with:
node-version: "24"
package-manager-cache: false
- name: Run Web UI task
- if: inputs.component == 'web-ui'
+ if: matrix.component == 'web-ui'
run: |
cd web
npm ci
- if [ "${{ inputs.task }}" = "lint" ]; then
+ if [ "${{ matrix.task }}" = "lint" ]; then
npm run lint
- elif [ "${{ inputs.task }}" = "test" ]; then
+ elif [ "${{ matrix.task }}" = "test" ]; then
npm test
- elif [ "${{ inputs.task }}" = "build" ]; then
+ elif [ "${{ matrix.task }}" = "build" ]; then
npm run build
fi
# Helm chart testing
- name: Setup Helm test tools
- if: inputs.component == 'helm' && (inputs.task == 'validate' || inputs.task == 'smoke')
+ if: matrix.component == 'helm' && (matrix.task == 'validate' || matrix.task == 'smoke')
uses: ./.github/actions/utils/setup-helm-tools
with:
- install-kind: ${{ inputs.task == 'smoke' }}
- install-helm-docs: ${{ inputs.task == 'validate' }}
- install-yamllint: ${{ inputs.task == 'validate' }}
- install-helmfmt: ${{ inputs.task == 'validate' }}
+ install-kind: ${{ matrix.task == 'smoke' }}
+ install-helm-docs: ${{ matrix.task == 'validate' }}
+ install-yamllint: ${{ matrix.task == 'validate' }}
+ install-helmfmt: ${{ matrix.task == 'validate' }}
- name: Setup Helm smoke cluster
- if: inputs.component == 'helm' && inputs.task == 'smoke'
+ if: matrix.component == 'helm' && matrix.task == 'smoke'
run: scripts/ci/setup-helm-smoke-cluster.sh
- name: Validate Helm chart
- if: inputs.component == 'helm' && inputs.task == 'validate'
+ if: matrix.component == 'helm' && matrix.task == 'validate'
run: scripts/ci/test-helm.sh validate
- name: Smoke test Helm chart runtime
- if: inputs.component == 'helm' && inputs.task == 'smoke'
+ if: matrix.component == 'helm' && matrix.task == 'smoke'
run: scripts/ci/test-helm.sh smoke --cleanup
- name: Collect Helm smoke diagnostics
- if: failure() && inputs.component == 'helm' && inputs.task == 'smoke'
+ if: failure() && matrix.component == 'helm' && matrix.task == 'smoke'
run: scripts/ci/test-helm.sh collect-smoke-diagnostics
# CI workflow validation
- name: Validate CI workflows
- if: inputs.component == 'ci-workflows' && inputs.task == 'validate'
+ if: matrix.component == 'ci-workflows' && matrix.task == 'validate'
run: |
echo "Validating GitHub Actions workflows..."
# Basic YAML validation
@@ -262,13 +261,14 @@ jobs:
python -c "import yaml; yaml.safe_load(open('$workflow'))" || exit 1
done
echo "All workflows are valid YAML"
+ python3 scripts/ci/test-runtime-artifacts.py
# Upload reports
- name: Upload reports
if: always()
uses: actions/upload-artifact@v7
with:
- name: ${{ inputs.component }}-${{ inputs.task }}-reports-${{ github.run_id }}-${{ github.run_attempt }}
+ name: ${{ matrix.component }}-${{ matrix.task }}-reports-${{ github.run_id }}-${{ github.run_attempt }}
path: |
reports/**
target/llvm-cov/**
diff --git a/.github/workflows/_test_bdd.yml b/.github/workflows/_test_bdd.yml
index 0e713c05d..db8dcbe81 100644
--- a/.github/workflows/_test_bdd.yml
+++ b/.github/workflows/_test_bdd.yml
@@ -19,18 +19,10 @@ name: _test_bdd
on:
workflow_call:
inputs:
- component:
+ matrix:
type: string
required: true
- description: "Component to test (e.g., 'bdd-rust', 'bdd-python', etc.)"
- task:
- type: string
- required: true
- description: "Task to run (e.g., 'bdd-rust', 'bdd-python', etc.)"
- timeout-minutes:
- type: number
- default: 60
- description: "Job timeout in minutes"
+ description: "Selected component/task entries"
secrets:
CODECOV_TOKEN:
@@ -42,103 +34,18 @@ permissions:
jobs:
run:
+ name: ${{ matrix.name || matrix.task }}
+ strategy:
+ fail-fast: true
+ matrix: ${{ fromJSON(inputs.matrix) }}
runs-on: ubuntu-26.04
- timeout-minutes: ${{ inputs.timeout-minutes }}
+ timeout-minutes: 60
steps:
- - name: Checkout code
- uses: actions/checkout@v7.0.1
-
- - name: Skip noop
- if: inputs.component == 'noop'
- run: echo "No changes detected, skipping tests"
-
- - name: Setup Rust for BDD
- if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-')
- uses: ./.github/actions/utils/setup-rust
- with:
- coverage: ${{ inputs.task == 'bdd-rust' }}
-
- - name: Build server for BDD tests
- if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-')
- run: |
- SERVER_BIN="iggy-server"
- echo "Building server binary and CLI for BDD tests..."
- cargo build --locked --bin iggy-server --bin iggy
-
- echo "Server binary built at: target/debug/${SERVER_BIN}"
- ls -lh "target/debug/${SERVER_BIN}"
-
- echo "CLI binary built at: target/debug/iggy"
- ls -lh target/debug/iggy
-
- # Verify the binary exists and is executable
- if [ ! -f "target/debug/${SERVER_BIN}" ]; then
- echo "ERROR: Server binary not found at target/debug/${SERVER_BIN}"
- exit 1
- fi
-
- if [ ! -f "target/debug/iggy" ]; then
- echo "ERROR: CLI binary not found at target/debug/iggy"
- exit 1
- fi
-
- - name: Run BDD tests
- if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-')
- run: |
- # Extract SDK name from task (format: bdd-<sdk>).
- SDK_NAME=$(echo "${{ inputs.task }}" | sed 's/^bdd-//')
- export IGGY_SERVER_PATH="target/debug/iggy-server"
- echo "Server binary location: $(ls -lh target/debug/iggy-server)"
-
- echo "Running BDD tests for SDK: $SDK_NAME"
- echo "Current directory: $(pwd)"
- echo "CLI binary location: $(ls -lh target/debug/iggy)"
-
- # Export path to the pre-built cli binary (relative to repo root)
- export IGGY_CLI_PATH="target/debug/iggy"
- export IGGY_ROOT_USERNAME="iggy"
- export IGGY_ROOT_PASSWORD="iggy"
- COVERAGE_ARGS=()
- if [[ "$SDK_NAME" == rust ]]; then
- COVERAGE_ARGS=(--coverage)
- fi
- ./scripts/run-bdd-tests.sh "${COVERAGE_ARGS[@]}" "$SDK_NAME"
-
- - name: Clean up Docker resources (BDD)
- if: always() && startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-')
- run: |
- ./scripts/run-bdd-tests.sh clean || true
- docker system prune -f || true
-
- - name: Generate Rust coverage report
- if: always() && !cancelled() && inputs.task == 'bdd-rust'
- run: |
- if compgen -G 'target/*.profraw' > /dev/null; then
- cargo llvm-cov report --codecov --output-path codecov.json
- else
- echo "::notice::No coverage profiles found; skipping report"
- fi
-
- - name: Upload Rust coverage to Codecov
- if: always() && !cancelled() && inputs.task == 'bdd-rust' && hashFiles('codecov.json', 'reports/rust-bdd-coverage.lcov') != ''
- uses: codecov/codecov-action@v7.1.1
- with:
- token: ${{ secrets.CODECOV_TOKEN }}
- files: codecov.json,reports/rust-bdd-coverage.lcov
- disable_search: true
- flags: rust-bdd
- fail_ci_if_error: false
- override_pr: ${{ github.event.pull_request.number }}
+ - uses: actions/checkout@v7.0.1
- - name: Upload reports
- if: always()
- uses: actions/upload-artifact@v7
+ - name: Run BDD suite
+ uses: ./.github/actions/bdd/pre-merge
with:
- name: ${{ inputs.component }}-${{ inputs.task }}-reports-${{ github.run_id }}-${{ github.run_attempt }}
- path: |
- reports/**
- target/llvm-cov/**
- coverage.lcov
- codecov.json
- if-no-files-found: ignore
- retention-days: 7
+ component: ${{ matrix.component }}
+ task: ${{ matrix.task }}
+ codecov-token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.github/workflows/_test_examples.yml b/.github/workflows/_test_examples.yml
index 6b3de5130..d05d2da80 100644
--- a/.github/workflows/_test_examples.yml
+++ b/.github/workflows/_test_examples.yml
@@ -19,14 +19,10 @@ name: _test_examples
on:
workflow_call:
inputs:
- component:
+ matrix:
type: string
required: true
- description: "Component to test (e.g., 'examples-rust', 'examples-go')"
- task:
- type: string
- required: true
- description: "Task to run (e.g., 'examples-rust', 'examples-go', 'examples-node')"
+ description: "Selected component/task entries"
permissions:
contents: read
@@ -34,29 +30,31 @@ permissions:
jobs:
run:
+ name: ${{ matrix.name || matrix.task }}
+ strategy:
+ fail-fast: true
+ matrix: ${{ fromJSON(inputs.matrix) }}
runs-on: ubuntu-26.04
timeout-minutes: 60
steps:
- name: Checkout code
uses: actions/checkout@v7.0.1
- - name: Skip noop
- if: inputs.component == 'noop'
- run: echo "No changes detected, skipping tests"
-
- name: Setup Rust for examples
- if: startsWith(inputs.component, 'examples-')
+ if: contains(fromJSON('["examples-rust","examples-python","examples-php"]'), matrix.task)
uses: ./.github/actions/utils/setup-rust
+ with:
+ free-disk-space: ${{ matrix.task != 'examples-python' }}
- name: Setup Node for examples
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-node'
+ if: matrix.task == 'examples-node'
uses: ./.github/actions/utils/setup-node
with:
node-version: "23"
workspace: examples/node
- name: Install PHP build dependencies
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-php'
+ if: matrix.task == 'examples-php'
shell: bash
run: |
./scripts/ci/apt-install.sh --no-install-recommends \
@@ -82,86 +80,53 @@ jobs:
"${php_config}" --version
- name: Setup Java for examples
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-java'
+ if: matrix.task == 'examples-java'
uses: ./.github/actions/utils/setup-java
with:
gradle-cache-disabled: "true"
- name: Setup Go for examples
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-go'
+ if: matrix.task == 'examples-go'
uses: ./.github/actions/utils/setup-go
with:
download-deps: "false"
- name: Test Go examples
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-go'
+ if: matrix.task == 'examples-go'
working-directory: examples/go
run: go test -race ./...
- name: Test Java examples
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-java'
+ if: matrix.task == 'examples-java'
working-directory: examples/java
run: ./gradlew test
- - name: Build common binaries for all examples
- if: startsWith(inputs.component, 'examples-')
- run: |
- echo "Building common binaries for all examples tests..."
- echo "Current directory: $(pwd)"
-
- # Every SDK speaks only the VSR wire protocol, so every lane runs
- # against the VSR server.
- SERVER_BIN="iggy-server"
- echo "Building ${SERVER_BIN}..."
- BUILD_ARGS=(--bin "$SERVER_BIN")
- if [[ "${{ inputs.task }}" == "examples-rust" ]]; then
- BUILD_ARGS+=(--bin iggy --examples)
- fi
- cargo build --locked "${BUILD_ARGS[@]}"
-
- # Verify server binary was built (needed by all examples)
- echo "Verifying server binary:"
- if [ -f "target/debug/${SERVER_BIN}" ]; then
- echo "✅ Server binary found at target/debug/${SERVER_BIN}"
- ls -lh "target/debug/${SERVER_BIN}"
- else
- echo "❌ Server binary NOT found at target/debug/${SERVER_BIN}"
- echo "Checking target directory structure:"
- find target -name "${SERVER_BIN}" -type f 2>/dev/null || true
- exit 1
- fi
+ - name: Build Rust examples
+ if: matrix.task == 'examples-rust'
+ run: cargo build --locked --examples
- # For Rust examples, verify CLI was built
- if [[ "${{ inputs.task }}" == "examples-rust" ]]; then
- if [ -f "target/debug/iggy" ]; then
- echo "✅ CLI binary found at target/debug/iggy"
- ls -lh target/debug/iggy
- else
- echo "❌ CLI binary NOT found at target/debug/iggy"
- exit 1
- fi
- fi
+ - name: Download runtime binaries
+ uses: ./.github/actions/utils/download-runtime
- # Keep Python's PKG_CONFIG_PATH out of the server build above.
- name: Setup Python
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python'
+ if: matrix.task == 'examples-python'
uses: actions/setup-python@v7.0.0
with:
python-version: "3.11"
- name: Setup uv
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python'
+ if: matrix.task == 'examples-python'
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
- name: Cache uv
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python'
+ if: matrix.task == 'examples-python'
uses: actions/cache@v6.1.0
with:
path: ~/.cache/uv
key: uv-${{ runner.os }}-${{ hashFiles('examples/python/uv.lock', 'foreign/python/uv.lock', 'bdd/python/uv.lock') }}
- name: Build PHP extension for examples
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-php'
+ if: matrix.task == 'examples-php'
shell: bash
run: |
export CARGO_TARGET_DIR="${GITHUB_WORKSPACE}/target"
@@ -175,7 +140,7 @@ jobs:
ls -lh "$extension"
- name: Build Node SDK for examples
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-node'
+ if: matrix.task == 'examples-node'
run: |
cd foreign/node
npm ci
@@ -186,38 +151,38 @@ jobs:
cd ../..
- name: Run Rust examples
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-rust'
+ if: matrix.task == 'examples-rust'
run: ./scripts/run-examples-from-readme.sh --language rust
- name: Run Go examples
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-go'
+ if: matrix.task == 'examples-go'
run: ./scripts/run-examples-from-readme.sh --language go
- name: Run C# examples
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-csharp'
+ if: matrix.task == 'examples-csharp'
run: ./scripts/run-examples-from-readme.sh --language csharp
- name: Run Python examples
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-python'
+ if: matrix.task == 'examples-python'
run: ./scripts/run-examples-from-readme.sh --language python
- name: Run PHP examples
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-php'
+ if: matrix.task == 'examples-php'
run: ./scripts/run-examples-from-readme.sh --language php
- name: Run Node.js examples
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-node'
+ if: matrix.task == 'examples-node'
run: ./scripts/run-examples-from-readme.sh --language node
- name: Run Java examples
- if: startsWith(inputs.component, 'examples-') && inputs.task == 'examples-java'
+ if: matrix.task == 'examples-java'
run: ./scripts/run-examples-from-readme.sh --language java
- name: Upload reports
if: always()
uses: actions/upload-artifact@v7
with:
- name: ${{ inputs.component }}-${{ inputs.task }}-reports-${{ github.run_id }}-${{ github.run_attempt }}
+ name: ${{ matrix.component }}-${{ matrix.task }}-reports-${{ github.run_id }}-${{ github.run_attempt }}
path: reports/**
if-no-files-found: ignore
retention-days: 7
diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml
index 6d838e6f4..aada7fa94 100644
--- a/.github/workflows/coverage-baseline.yml
+++ b/.github/workflows/coverage-baseline.yml
@@ -79,41 +79,58 @@ jobs:
>> "$GITHUB_OUTPUT"
shell: bash
- rust-coverage:
- name: Rust coverage baseline
+ runtime:
+ name: Runtime binaries
+ needs: detect
+ if: contains(toJSON(needs.detect.outputs), 'true')
+ uses: ./.github/workflows/_build_ci_artifacts.yml
+ with:
+ task: runtime
+ save-cache: true
+ build-metadata: true
+
+ rust-test-build:
+ name: Rust test artifacts
needs: detect
if: needs.detect.outputs.rust == 'true'
- permissions:
- contents: read
- issues: read
- uses: ./.github/workflows/_rust.yml
+ uses: ./.github/workflows/_build_ci_artifacts.yml
with:
- plan: '{"checks":[],"tests":true}'
+ task: test-build
test-scope: workspace
- timeout-minutes: 120
save-cache: true
build-metadata: true
- secrets:
- CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- rust-bdd-coverage:
- name: Rust BDD coverage baseline
- needs: detect
+ rust-coverage:
+ name: Rust
+ needs: [detect, runtime, rust-test-build]
if: needs.detect.outputs.rust == 'true'
permissions:
contents: read
- security-events: write
- uses: ./.github/workflows/_test_bdd.yml
+ uses: ./.github/workflows/_rust_tests.yml
with:
- component: bdd-rust
- task: bdd-rust
timeout-minutes: 120
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+ rust-bdd-coverage:
+ name: Rust BDD
+ needs: [detect, runtime]
+ if: needs.detect.outputs.rust == 'true'
+ runs-on: ubuntu-26.04
+ timeout-minutes: 120
+ steps:
+ - uses: actions/checkout@v7.0.1
+
+ - name: Run Rust BDD with coverage
+ uses: ./.github/actions/bdd/pre-merge
+ with:
+ component: bdd-rust
+ task: bdd-rust
+ codecov-token: ${{ secrets.CODECOV_TOKEN }}
+
java-coverage:
- name: Java coverage baseline
- needs: detect
+ name: Java
+ needs: [detect, runtime]
if: needs.detect.outputs.java == 'true'
runs-on: ubuntu-26.04
timeout-minutes: 120
@@ -123,14 +140,14 @@ jobs:
- name: Setup Java
uses: ./.github/actions/utils/setup-java
- - name: Setup Rust
- uses: ./.github/actions/utils/setup-rust
- with:
- save-cache: "false"
+ - name: Download runtime binaries
+ uses: ./.github/actions/utils/download-runtime
- name: Start Iggy server
id: iggy
uses: ./.github/actions/utils/server-start
+ with:
+ bin: target/debug/iggy-server
- name: Run tests and generate coverage
working-directory: foreign/java
@@ -157,15 +174,11 @@ jobs:
fail_ci_if_error: false
csharp-coverage:
- name: C# coverage baseline (${{ matrix.framework }})
- needs: detect
+ name: C#
+ needs: [detect, runtime]
if: needs.detect.outputs.csharp == 'true'
runs-on: ubuntu-26.04
timeout-minutes: 120
- strategy:
- fail-fast: false
- matrix:
- framework: [net8.0, net10.0]
steps:
- uses: actions/checkout@v7.0.1
@@ -176,15 +189,14 @@ jobs:
8.0.x
10.0.x
- - name: Setup Rust
- uses: ./.github/actions/utils/setup-rust
+ - name: Download runtime binaries
+ uses: ./.github/actions/utils/download-runtime
with:
- save-cache: "false"
+ install-system-dependencies: "false"
- name: Build Iggy server Docker image
id: docker_build
run: |
- cargo build --locked --bin iggy-server --bin iggy
docker build \
-f core/server/Dockerfile \
--target runtime-prebuilt \
@@ -195,55 +207,66 @@ jobs:
echo "docker_image=iggy-server:test" >> "$GITHUB_OUTPUT"
- name: Restore and build
+ id: build
working-directory: foreign/csharp
run: |
dotnet restore Iggy_SDK.sln
- dotnet build Iggy_SDK_Tests --no-restore --framework ${{ matrix.framework }}
- dotnet build Iggy_SDK.Tests.Integration --no-restore --framework ${{ matrix.framework }}
+ dotnet build Iggy_SDK_Tests --no-restore
+ dotnet build Iggy_SDK.Tests.Integration --no-restore
- name: Run unit tests with coverage
working-directory: foreign/csharp
run: |
- dotnet test \
- --project Iggy_SDK_Tests \
- --framework ${{ matrix.framework }} \
- --no-build \
- --verbosity normal \
- --coverage \
- --coverage-output-format cobertura \
- --coverage-output coverage.cobertura.xml \
- --results-directory ./reports/unit
+ result=0
+ for framework in net8.0 net10.0; do
+ dotnet test \
+ --project Iggy_SDK_Tests \
+ --framework "$framework" \
+ --no-build \
+ --verbosity normal \
+ --coverage \
+ --coverage-output-format cobertura \
+ --coverage-output coverage.cobertura.xml \
+ --results-directory ./reports/unit/$framework || result=$?
+ done
+ exit "$result"
- name: Run integration tests with coverage
+ if: ${{ !cancelled() && steps.build.outcome == 'success' }}
working-directory: foreign/csharp
env:
IGGY_SERVER_DOCKER_IMAGE: ${{ steps.docker_build.outputs.docker_image }}
IGGY_TEST_LOGS_DIR: ./reports/container-logs
IGGY_TEST_CLUSTER_NODES: "3"
run: |
- dotnet test \
- --project Iggy_SDK.Tests.Integration \
- --framework ${{ matrix.framework }} \
- --no-build \
- --verbosity normal \
- --coverage \
- --coverage-output-format cobertura \
- --coverage-output coverage.cobertura.xml \
- --results-directory ./reports/integration \
- -- --retry-failed-tests 3
+ result=0
+ for framework in net8.0 net10.0; do
+ dotnet test \
+ --project Iggy_SDK.Tests.Integration \
+ --framework "$framework" \
+ --no-build \
+ --verbosity normal \
+ --coverage \
+ --coverage-output-format cobertura \
+ --coverage-output coverage.cobertura.xml \
+ --results-directory ./reports/integration/$framework \
+ -- --retry-failed-tests 3 || result=$?
+ done
+ exit "$result"
- name: Upload to Codecov
+ if: ${{ !cancelled() && steps.build.outcome == 'success' }}
uses: codecov/codecov-action@v7.1.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
- files: foreign/csharp/reports/unit/coverage.cobertura.xml,foreign/csharp/reports/integration/coverage.cobertura.xml
+ files: foreign/csharp/reports/unit/net8.0/coverage.cobertura.xml,foreign/csharp/reports/unit/net10.0/coverage.cobertura.xml,foreign/csharp/reports/integration/net8.0/coverage.cobertura.xml,foreign/csharp/reports/integration/net10.0/coverage.cobertura.xml
disable_search: true
flags: csharp
fail_ci_if_error: false
python-coverage:
- name: Python coverage baseline
- needs: detect
+ name: Python
+ needs: [detect, runtime]
if: needs.detect.outputs.python == 'true'
runs-on: ubuntu-26.04
timeout-minutes: 120
@@ -255,12 +278,13 @@ jobs:
with:
save-cache: "false"
- # setup-python changes PKG_CONFIG_PATH and invalidates cached native server dependencies.
+ - name: Download runtime binaries
+ uses: ./.github/actions/utils/download-runtime
+
- name: Build server Docker image for TLS tests
run: |
# test_tls.py spawns the server in a container; build it from the
# same binaries the plain-TCP tests run against.
- cargo build --locked --bin iggy-server --bin iggy
docker build \
-f core/server/Dockerfile \
--target runtime-prebuilt \
@@ -273,7 +297,6 @@ jobs:
id: iggy
uses: ./.github/actions/utils/server-start
with:
- mode: bin
bin: target/debug/iggy-server
- name: Setup Python
@@ -368,8 +391,8 @@ jobs:
fail_ci_if_error: false
php-coverage:
- name: PHP coverage baseline
- needs: detect
+ name: PHP
+ needs: [detect, runtime]
if: needs.detect.outputs.php == 'true'
runs-on: ubuntu-26.04
timeout-minutes: 120
@@ -391,8 +414,8 @@ jobs:
fail_ci_if_error: false
node-coverage:
- name: Node coverage baseline
- needs: detect
+ name: Node
+ needs: [detect, runtime]
if: needs.detect.outputs.node == 'true'
runs-on: ubuntu-26.04
timeout-minutes: 120
@@ -408,10 +431,8 @@ jobs:
- name: Pin npm to 11 to match Dependabot lockfile format
run: npm install -g npm@11
- - name: Setup Rust
- uses: ./.github/actions/utils/setup-rust
- with:
- save-cache: "false"
+ - name: Download runtime binaries
+ uses: ./.github/actions/utils/download-runtime
- name: Install netcat
run: ./scripts/ci/apt-install.sh netcat-openbsd
@@ -425,7 +446,7 @@ jobs:
id: iggy
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
wait-timeout-seconds: "90"
- name: Run unit tests with coverage
@@ -457,8 +478,8 @@ jobs:
fail_ci_if_error: false
go-coverage:
- name: Go coverage baseline
- needs: detect
+ name: Go
+ needs: [detect, runtime]
if: needs.detect.outputs.go == 'true'
runs-on: ubuntu-26.04
timeout-minutes: 120
@@ -468,10 +489,8 @@ jobs:
- name: Setup Go
uses: ./.github/actions/utils/setup-go
- - name: Setup Rust
- uses: ./.github/actions/utils/setup-rust
- with:
- save-cache: "false"
+ - name: Download runtime binaries
+ uses: ./.github/actions/utils/download-runtime
# The Go SDK speaks the VSR wire protocol, so its coverage run needs the
# VSR server rather than the legacy one.
@@ -479,7 +498,7 @@ jobs:
id: iggy
uses: ./.github/actions/utils/server-start
with:
- cargo-bin: iggy-server
+ bin: target/debug/iggy-server
replica-id: "0"
wait-timeout-seconds: "90"
@@ -545,8 +564,8 @@ jobs:
fail_ci_if_error: false
cpp-coverage:
- name: C++ coverage baseline
- needs: detect
+ name: C++
+ needs: [detect, runtime]
if: needs.detect.outputs.cpp == 'true'
runs-on: ubuntu-26.04
timeout-minutes: 120
diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml
index bcb96cf60..13456e635 100644
--- a/.github/workflows/pre-merge.yml
+++ b/.github/workflows/pre-merge.yml
@@ -31,7 +31,7 @@ concurrency:
permissions:
contents: read
- # listLabelsOnIssue in _test.yml, for the breaking:storage escape hatch
+ # listLabelsOnIssue in _rust.yml, for the breaking:storage escape hatch
issues: read
security-events: write
pull-requests: write
@@ -53,6 +53,24 @@ jobs:
name: Detect changes
uses: ./.github/workflows/_detect.yml
+ runtime:
+ name: Runtime binaries
+ needs: detect
+ if: needs.detect.outputs.runtime == 'true'
+ uses: ./.github/workflows/_build_ci_artifacts.yml
+ with:
+ task: runtime
+ timeout-minutes: 60
+
+ rust-test-build:
+ name: Rust test artifacts
+ needs: detect
+ if: needs.detect.outputs.rust_runtime_plan != '' && fromJSON(needs.detect.outputs.rust_runtime_plan).tests
+ uses: ./.github/workflows/_build_ci_artifacts.yml
+ with:
+ task: test-build
+ timeout-minutes: 60
+
# Rust components
test-rust:
name: Rust
@@ -61,173 +79,138 @@ jobs:
uses: ./.github/workflows/_rust.yml
with:
plan: ${{ needs.detect.outputs.rust_plan }}
+
+ test-rust-runtime:
+ name: Rust tests
+ needs: [detect, runtime, rust-test-build]
+ if: ${{ !cancelled() && !failure() && needs.detect.outputs.rust_runtime_plan != '' }}
+ uses: ./.github/workflows/_rust.yml
+ with:
+ plan: ${{ needs.detect.outputs.rust_runtime_plan }}
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
# Python SDK
test-python:
- name: Python • ${{ matrix.task }}
- needs: detect
- if: ${{ fromJson(needs.detect.outputs.python_matrix).include[0].component != 'noop' }}
- strategy:
- fail-fast: true
- matrix: ${{ fromJson(needs.detect.outputs.python_matrix) }}
+ name: Python
+ needs: [detect, runtime]
+ if: ${{ !cancelled() && !failure() && fromJson(needs.detect.outputs.python_matrix).include[0].component != 'noop' }}
uses: ./.github/workflows/_test.yml
with:
- component: ${{ matrix.component }}
- task: ${{ matrix.task }}
+ matrix: ${{ needs.detect.outputs.python_matrix }}
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
# PHP SDK
test-php:
- name: PHP • ${{ matrix.task }}
- needs: detect
- if: ${{ fromJson(needs.detect.outputs.php_matrix).include[0].component != 'noop' }}
- strategy:
- fail-fast: false
- matrix: ${{ fromJson(needs.detect.outputs.php_matrix) }}
+ name: PHP
+ needs: [detect, runtime]
+ if: ${{ !cancelled() && !failure() && fromJson(needs.detect.outputs.php_matrix).include[0].component != 'noop' }}
uses: ./.github/workflows/_test.yml
with:
- component: ${{ matrix.component }}
- task: ${{ matrix.task }}
+ matrix: ${{ needs.detect.outputs.php_matrix }}
+ fail-fast: false
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
# Node SDK
test-node:
- name: Node • ${{ matrix.task }}
- needs: detect
- if: ${{ fromJson(needs.detect.outputs.node_matrix).include[0].component != 'noop' }}
- strategy:
- fail-fast: true
- matrix: ${{ fromJson(needs.detect.outputs.node_matrix) }}
+ name: Node
+ needs: [detect, runtime]
+ if: ${{ !cancelled() && !failure() && fromJson(needs.detect.outputs.node_matrix).include[0].component != 'noop' }}
uses: ./.github/workflows/_test.yml
with:
- component: ${{ matrix.component }}
- task: ${{ matrix.task }}
+ matrix: ${{ needs.detect.outputs.node_matrix }}
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
# Go SDK
test-go:
- name: Go • ${{ matrix.task }}
- needs: detect
- if: ${{ fromJson(needs.detect.outputs.go_matrix).include[0].component != 'noop' }}
- strategy:
- fail-fast: true
- matrix: ${{ fromJson(needs.detect.outputs.go_matrix) }}
+ name: Go
+ needs: [detect, runtime]
+ if: ${{ !cancelled() && !failure() && fromJson(needs.detect.outputs.go_matrix).include[0].component != 'noop' }}
uses: ./.github/workflows/_test.yml
with:
- component: ${{ matrix.component }}
- task: ${{ matrix.task }}
+ matrix: ${{ needs.detect.outputs.go_matrix }}
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
# Java SDK
test-java:
- name: Java • ${{ matrix.task }}
- needs: detect
- if: ${{ fromJson(needs.detect.outputs.java_matrix).include[0].component != 'noop' }}
- strategy:
- fail-fast: true
- matrix: ${{ fromJson(needs.detect.outputs.java_matrix) }}
+ name: Java
+ needs: [detect, runtime]
+ if: ${{ !cancelled() && !failure() && fromJson(needs.detect.outputs.java_matrix).include[0].component != 'noop' }}
uses: ./.github/workflows/_test.yml
with:
- component: ${{ matrix.component }}
- task: ${{ matrix.task }}
+ matrix: ${{ needs.detect.outputs.java_matrix }}
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
# C# SDK
test-csharp:
- name: C# • ${{ matrix.task }}
- needs: detect
- if: ${{ fromJson(needs.detect.outputs.csharp_matrix).include[0].component != 'noop' }}
- strategy:
- fail-fast: true
- matrix: ${{ fromJson(needs.detect.outputs.csharp_matrix) }}
+ name: C#
+ needs: [detect, runtime]
+ if: ${{ !cancelled() && !failure() && fromJson(needs.detect.outputs.csharp_matrix).include[0].component != 'noop' }}
uses: ./.github/workflows/_test.yml
with:
- component: ${{ matrix.component }}
- task: ${{ matrix.task }}
+ matrix: ${{ needs.detect.outputs.csharp_matrix }}
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
# C++ SDK
test-cpp:
- name: C++ • ${{ matrix.task }}
- needs: detect
- if: ${{ fromJson(needs.detect.outputs.cpp_matrix).include[0].component != 'noop' }}
- strategy:
- fail-fast: true
- matrix: ${{ fromJson(needs.detect.outputs.cpp_matrix) }}
+ name: C++
+ needs: [detect, runtime]
+ if: ${{ !cancelled() && !failure() && fromJson(needs.detect.outputs.cpp_matrix).include[0].component != 'noop' }}
uses: ./.github/workflows/_test.yml
with:
- component: ${{ matrix.component }}
- task: ${{ matrix.task }}
+ matrix: ${{ needs.detect.outputs.cpp_matrix }}
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
# Swift SDK
test-swift:
- name: Swift • ${{ matrix.task }}
- needs: detect
- if: ${{ fromJson(needs.detect.outputs.swift_matrix).include[0].component != 'noop' }}
- strategy:
- fail-fast: true
- matrix: ${{ fromJson(needs.detect.outputs.swift_matrix) }}
+ name: Swift
+ needs: [detect, runtime]
+ if: ${{ !cancelled() && !failure() && fromJson(needs.detect.outputs.swift_matrix).include[0].component != 'noop' }}
uses: ./.github/workflows/_test.yml
with:
- component: ${{ matrix.component }}
- task: ${{ matrix.task }}
+ matrix: ${{ needs.detect.outputs.swift_matrix }}
# Other components
test-other:
- name: Other • ${{ matrix.component }}/${{ matrix.task }}
+ name: Other
needs: detect
if: ${{ fromJson(needs.detect.outputs.other_matrix).include[0].component != 'noop' }}
- strategy:
- fail-fast: true
- matrix: ${{ fromJson(needs.detect.outputs.other_matrix) }}
uses: ./.github/workflows/_test.yml
with:
- component: ${{ matrix.component }}
- task: ${{ matrix.task }}
+ matrix: ${{ needs.detect.outputs.other_matrix }}
# BDD Tests
test-bdd:
- name: BDD • ${{ matrix.component }}/${{ matrix.task }}
- needs: detect
- if: ${{ fromJson(needs.detect.outputs.bdd_matrix).include[0].component != 'noop' }}
- strategy:
- fail-fast: true
- matrix: ${{ fromJson(needs.detect.outputs.bdd_matrix) }}
+ name: BDD
+ needs: [detect, runtime]
+ if: ${{ !cancelled() && !failure() && fromJson(needs.detect.outputs.bdd_matrix).include[0].component != 'noop' }}
uses: ./.github/workflows/_test_bdd.yml
with:
- component: ${{ matrix.component }}
- task: ${{ matrix.task }}
+ matrix: ${{ needs.detect.outputs.bdd_matrix }}
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
# Examples Tests
test-examples:
- name: Examples • ${{ matrix.component }}/${{ matrix.task }}
- needs: detect
- if: ${{ fromJson(needs.detect.outputs.examples_matrix).include[0].component != 'noop' }}
- strategy:
- fail-fast: true
- matrix: ${{ fromJson(needs.detect.outputs.examples_matrix) }}
+ name: Examples
+ needs: [detect, runtime]
+ if: ${{ !cancelled() && !failure() && fromJson(needs.detect.outputs.examples_matrix).include[0].component != 'noop' }}
uses: ./.github/workflows/_test_examples.yml
with:
- component: ${{ matrix.component }}
- task: ${{ matrix.task }}
+ matrix: ${{ needs.detect.outputs.examples_matrix }}
# Final status check
finalize_pr:
runs-on: ubuntu-26.04-arm
timeout-minutes: 5
- needs: [common, detect, test-rust, test-python, test-php, test-node, test-go, test-java, test-csharp, test-cpp, test-swift, test-bdd, test-examples, test-other]
+ needs: [common, detect, runtime, rust-test-build, test-rust, test-rust-runtime, test-python, test-php, test-node, test-go, test-java, test-csharp, test-cpp, test-swift, test-bdd, test-examples, test-other]
if: ${{ !cancelled() }}
steps:
- name: Get job execution times
@@ -242,109 +225,27 @@ jobs:
per_page: 100
});
- const jobTimes = {};
- const jobStatuses = {};
-
- const formatDuration = (ms) => {
+ const formatDuration = ms => {
const seconds = Math.floor(ms / 1000);
- const minutes = Math.floor(seconds / 60);
- const hours = Math.floor(minutes / 60);
- if (hours > 0) {
- return `${hours}h ${minutes % 60}m ${seconds % 60}s`;
- } else if (minutes > 0) {
- return `${minutes}m ${seconds % 60}s`;
- } else if (seconds > 0) {
- return `${seconds}s`;
- } else {
- return '< 1s';
- }
+ return `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
};
-
- // Log job names for debugging
- console.log('Job names found:');
- for (const job of jobs) {
- console.log(` - ${job.name}: ${job.status} (conclusion: ${job.conclusion || 'N/A'})`);
- jobStatuses[job.name] = job.conclusion || job.status;
-
- // Only show duration for jobs that actually ran
- if (job.started_at && job.completed_at) {
- const start = new Date(job.started_at);
- const end = new Date(job.completed_at);
- const duration = end - start;
- jobTimes[job.name] = formatDuration(duration);
- } else if (job.started_at && !job.completed_at) {
- // Job is still running
- const start = new Date(job.started_at);
- const duration = Date.now() - start;
- jobTimes[job.name] = formatDuration(duration) + ' ⏳';
- } else {
- // Job was skipped or hasn't started
- jobTimes[job.name] = null;
- }
- }
-
- // Helper to find job info for a component
- const findJobInfo = (prefix) => {
- for (const [name, status] of Object.entries(jobStatuses)) {
- if (name.startsWith(prefix)) {
- return {
- time: jobTimes[name],
- status: status
- };
- }
- }
- return { time: null, status: 'skipped' };
+ const groups = {
+ rust: 'Rust', rust_tests: 'Rust tests', python: 'Python', php: 'PHP', node: 'Node',
+ go: 'Go', java: 'Java', csharp: 'C#', cpp: 'C++', swift: 'Swift',
+ bdd: 'BDD', examples: 'Examples', other: 'Other',
+ common: 'Common checks', detect: 'Detect changes', runtime: 'Runtime binaries',
+ rust_test_build: 'Rust test artifacts',
};
-
- // Format duration based on job status
- const formatJobDuration = (info) => {
- if (info.status === 'skipped') return '-';
- if (info.time === null) return '-';
- return info.time;
- };
-
- // Set outputs for each component
- const python = findJobInfo('Python •');
- const php = findJobInfo('PHP •');
- const node = findJobInfo('Node •');
- const go = findJobInfo('Go •');
- const java = findJobInfo('Java •');
- const csharp = findJobInfo('C# •');
- const cpp = findJobInfo('C++ •');
- const swift = findJobInfo('Swift •');
- const bdd = findJobInfo('BDD •');
- const examples = findJobInfo('Examples •');
- const other = findJobInfo('Other •');
-
- // For non-matrix jobs, check by exact name
- const common = jobStatuses['Common checks']
- ? { time: jobTimes['Common checks'], status: jobStatuses['Common checks'] }
- : { time: null, status: 'skipped' };
- const detect = jobStatuses['Detect changes']
- ? { time: jobTimes['Detect changes'], status: jobStatuses['Detect changes'] }
- : { time: null, status: 'skipped' };
-
- // Output formatted durations
- const rustJobs = jobs.filter(job => job.name.startsWith('Rust /') &&
- job.conclusion !== 'skipped' && job.started_at && job.completed_at);
- const rustDuration = rustJobs.length ? formatDuration(
- Math.max(...rustJobs.map(job => new Date(job.completed_at).getTime())) -
- Math.min(...rustJobs.map(job => new Date(job.started_at).getTime()))
- ) : '-';
- core.setOutput('rust_time', rustDuration);
- core.setOutput('python_time', formatJobDuration(python));
- core.setOutput('php_time', formatJobDuration(php));
- core.setOutput('node_time', formatJobDuration(node));
- core.setOutput('go_time', formatJobDuration(go));
- core.setOutput('java_time', formatJobDuration(java));
- core.setOutput('csharp_time', formatJobDuration(csharp));
- core.setOutput('cpp_time', formatJobDuration(cpp));
- core.setOutput('swift_time', formatJobDuration(swift));
- core.setOutput('bdd_time', formatJobDuration(bdd));
- core.setOutput('examples_time', formatJobDuration(examples));
- core.setOutput('other_time', formatJobDuration(other));
- core.setOutput('common_time', formatJobDuration(common));
- core.setOutput('detect_time', formatJobDuration(detect));
+ for (const [key, group] of Object.entries(groups)) {
+ const completed = jobs.filter(job =>
+ (job.name === group || job.name.startsWith(`${group} /`)) &&
+ job.conclusion !== 'skipped' && job.started_at && job.completed_at);
+ const duration = completed.length ? formatDuration(
+ Math.max(...completed.map(job => new Date(job.completed_at).getTime())) -
+ Math.min(...completed.map(job => new Date(job.started_at).getTime()))
+ ) : '-';
+ core.setOutput(`${key}_time`, duration);
+ }
// Calculate total time - find the earliest job start time
let earliestStart = null;
@@ -374,7 +275,6 @@ jobs:
core.setOutput('total_time', '-');
}
- return jobTimes;
- name: Check status
run: |
@@ -422,8 +322,14 @@ jobs:
echo "| 🔍 Detection | $detect_status | ${{ steps.times.outputs.detect_time }} |" >> "$GITHUB_STEP_SUMMARY"
echo "| 📋 Common Checks | $common_status | ${{ steps.times.outputs.common_time }} |" >> "$GITHUB_STEP_SUMMARY"
+ runtime_status=$(format_status "${{ needs.runtime.result }}" "${{ steps.times.outputs.runtime_time }}")
+ echo "| Runtime binaries | $runtime_status | ${{ steps.times.outputs.runtime_time }} |" >> "$GITHUB_STEP_SUMMARY"
+ rust_test_build_status=$(format_status "${{ needs.rust-test-build.result }}" "${{ steps.times.outputs.rust_test_build_time }}")
+ echo "| Rust test artifacts | $rust_test_build_status | ${{ steps.times.outputs.rust_test_build_time }} |" >> "$GITHUB_STEP_SUMMARY"
+
# Language/component tests
rust_status=$(format_status "${{ needs.test-rust.result }}" "${{ steps.times.outputs.rust_time }}")
+ rust_tests_status=$(format_status "${{ needs.test-rust-runtime.result }}" "${{ steps.times.outputs.rust_tests_time }}")
python_status=$(format_status "${{ needs.test-python.result }}" "${{ steps.times.outputs.python_time }}")
php_status=$(format_status "${{ needs.test-php.result }}" "${{ steps.times.outputs.php_time }}")
node_status=$(format_status "${{ needs.test-node.result }}" "${{ steps.times.outputs.node_time }}")
@@ -437,6 +343,7 @@ jobs:
other_status=$(format_status "${{ needs.test-other.result }}" "${{ steps.times.outputs.other_time }}")
echo "| 🦀 Rust | $rust_status | ${{ steps.times.outputs.rust_time }} |" >> "$GITHUB_STEP_SUMMARY"
+ echo "| 🦀 Rust tests | $rust_tests_status | ${{ steps.times.outputs.rust_tests_time }} |" >> "$GITHUB_STEP_SUMMARY"
echo "| 🐍 Python | $python_status | ${{ steps.times.outputs.python_time }} |" >> "$GITHUB_STEP_SUMMARY"
echo "| 🐘 PHP | $php_status | ${{ steps.times.outputs.php_time }} |" >> "$GITHUB_STEP_SUMMARY"
echo "| 🟢 Node | $node_status | ${{ steps.times.outputs.node_time }} |" >> "$GITHUB_STEP_SUMMARY"
@@ -452,37 +359,11 @@ jobs:
echo "**Total workflow time:** ${{ steps.times.outputs.total_time }}" >> "$GITHUB_STEP_SUMMARY"
# Check for failures
- if [[ "${{ needs.common.result }}" == "failure" ]] || \
- [[ "${{ needs.detect.result }}" == "failure" ]] || \
- [[ "${{ needs.test-rust.result }}" == "failure" ]] || \
- [[ "${{ needs.test-python.result }}" == "failure" ]] || \
- [[ "${{ needs.test-php.result }}" == "failure" ]] || \
- [[ "${{ needs.test-node.result }}" == "failure" ]] || \
- [[ "${{ needs.test-go.result }}" == "failure" ]] || \
- [[ "${{ needs.test-java.result }}" == "failure" ]] || \
- [[ "${{ needs.test-csharp.result }}" == "failure" ]] || \
- [[ "${{ needs.test-cpp.result }}" == "failure" ]] || \
- [[ "${{ needs.test-swift.result }}" == "failure" ]] || \
- [[ "${{ needs.test-bdd.result }}" == "failure" ]] || \
- [[ "${{ needs.test-examples.result }}" == "failure" ]] || \
- [[ "${{ needs.test-other.result }}" == "failure" ]]; then
+ if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "❌ **CI Failed** - Please check the logs for details." >> "$GITHUB_STEP_SUMMARY"
exit 1
- elif [[ "${{ needs.common.result }}" == "cancelled" ]] || \
- [[ "${{ needs.detect.result }}" == "cancelled" ]] || \
- [[ "${{ needs.test-rust.result }}" == "cancelled" ]] || \
- [[ "${{ needs.test-python.result }}" == "cancelled" ]] || \
- [[ "${{ needs.test-php.result }}" == "cancelled" ]] || \
- [[ "${{ needs.test-node.result }}" == "cancelled" ]] || \
- [[ "${{ needs.test-go.result }}" == "cancelled" ]] || \
- [[ "${{ needs.test-java.result }}" == "cancelled" ]] || \
- [[ "${{ needs.test-csharp.result }}" == "cancelled" ]] || \
- [[ "${{ needs.test-cpp.result }}" == "cancelled" ]] || \
- [[ "${{ needs.test-swift.result }}" == "cancelled" ]] || \
- [[ "${{ needs.test-bdd.result }}" == "cancelled" ]] || \
- [[ "${{ needs.test-examples.result }}" == "cancelled" ]] || \
- [[ "${{ needs.test-other.result }}" == "cancelled" ]]; then
+ elif [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "⚠️ **CI Cancelled** - The workflow was cancelled." >> "$GITHUB_STEP_SUMMARY"
exit 1
diff --git a/AGENTS.md b/AGENTS.md
index e8019e34f..47f65a50f 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -233,7 +233,7 @@ cargo test -p integration -- connectors::runtime::benchmark::given_logging_forma
- **`test_logs/` grows quickly.** Wipe between major refactors.
- **Miri only covers `binary_protocol` + `consensus`.** Cannot emulate `io_uring` syscalls. do not try to expand Miri to crates that pull `compio`.
- **Integration crate has no `--test` target.** Filter by test path inside the single `mod.rs` binary.
-- **The integration harness runs prebuilt binaries and never builds them.** `Command::cargo_bin` only resolves a path, so a stale `iggy-server`, `iggy-connectors` or `iggy-mcp` is used silently. CI builds first (`cargo build --locked --bin iggy-server --bin iggy`); a local run does not, and a stale binary fails in ways that look like the change under test. Rebuild after anything that touches a launched binary or its config.
+- **The integration harness runs prebuilt binaries and never builds them.** `Command::cargo_bin` only resolves a path, so a stale `iggy-server`, `iggy-connectors` or `iggy-mcp` is used silently. CI restores revision-checked artifacts from the runtime and Rust test builds; a local run does not build them, and a stale binary fails in ways that look like the change under test. Rebuild after anything that touches a launched binary or its config.
## Local state directories
diff --git a/scripts/ci/restore-runtime.sh b/scripts/ci/restore-runtime.sh
new file mode 100755
index 000000000..d989a8ae8
--- /dev/null
+++ b/scripts/ci/restore-runtime.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+artifact_dir="${1:?usage: restore-runtime.sh <artifact-directory>}"
+revision=$(git rev-parse HEAD)
+artifact_revision=$(cat "$artifact_dir/revision")
+if [[ "$artifact_revision" != "$revision" ]]; then
+ echo "::error::Runtime revision $artifact_revision does not match checkout $revision"
+ exit 1
+fi
+
+tar --zstd -xf "$artifact_dir/runtime.tar.zst"
+for binary in target/debug/iggy-server target/debug/iggy; do
+ if [[ ! -x "$binary" ]]; then
+ echo "::error::Runtime binary missing or not executable: $binary"
+ exit 1
+ fi
+done
diff --git a/scripts/ci/storage-compat.sh b/scripts/ci/storage-compat.sh
index 296893268..86dca6c52 100755
--- a/scripts/ci/storage-compat.sh
+++ b/scripts/ci/storage-compat.sh
@@ -23,30 +23,27 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh"
# storage-compat.sh -- Prove HEAD still reads a data directory written by master.
#
-# Builds two iggy-server binaries and hands both to one integration test:
+# Supplies two iggy-server binaries to one integration test:
# baseline built from --baseline-ref, copied aside. Default: on a
# pull_request run, the master tip under the checked-out merge ref;
# anywhere else, origin/master.
-# HEAD built from the working tree, left at target/debug/iggy-server
+# HEAD restored from the CI runtime archive, or built locally from HEAD
# The test boots the baseline, seeds a data directory, swaps the binary to HEAD
# and restarts against that same directory.
#
# The baseline's own core/server/config.toml is extracted next to its binary and
# both boots read it, so the swap changes the binary and nothing else.
#
-# Both binaries MUST be built here. `core/integration` has no dependency on the
-# `server` package; the harness only LOCATES a binary, through
-# `assert_cmd::Command::cargo_bin`, which falls back to whatever file happens to
-# sit at target/debug/iggy-server (assert_cmd 2.2.2 `legacy_cargo_bin`). So
-# `cargo nextest run -p integration` compiles no server at all, and a lane that
-# does not build both halves compares a stale binary against itself and reports
-# green forever.
+# The harness only locates binaries; it never builds them. CI supplies HEAD
+# through a revision-checked runtime archive. Local runs build HEAD here.
+# The baseline is always resolved independently from its own source tree.
#
# Exit codes: 0 = compatible, non-zero = build failure or format regression.
# Empty selects the default described above once HEAD is known.
BASELINE_REF=""
REBUILD_BASELINE=0
+RUNTIME_ARTIFACTS=""
# The single #[ignore]d test this script exists to drive. The integration crate
# is one binary built from tests/mod.rs, so tests are selected by module path.
@@ -62,17 +59,23 @@ while [[ $# -gt 0 ]]; do
fi
shift 2
;;
+ --runtime-artifacts)
+ RUNTIME_ARTIFACTS="${2:?--runtime-artifacts requires an archive directory}"
+ RUNTIME_ARTIFACTS="$(realpath "$RUNTIME_ARTIFACTS")"
+ shift 2
+ ;;
--rebuild-baseline)
REBUILD_BASELINE=1
shift
;;
--help|-h)
- echo "Usage: $0 [--baseline-ref <ref>] [--rebuild-baseline]"
+ echo "Usage: $0 [--baseline-ref <ref>] [--rebuild-baseline] [--runtime-artifacts <directory>]"
echo ""
echo "Options:"
- echo " --baseline-ref <ref> Ref to build the baseline server from (default: the merge ref's"
- echo " first parent on a pull_request run, origin/master otherwise)"
- echo " --rebuild-baseline Rebuild the baseline even when its binary is already on disk"
+ echo " --baseline-ref <ref> Ref to build the baseline server from (default: the merge ref's"
+ echo " first parent on a pull_request run, origin/master otherwise)"
+ echo " --runtime-artifacts <directory> Use a revision-checked CI runtime archive for HEAD"
+ echo " --rebuild-baseline Rebuild the baseline even when its binary is already on disk"
exit 0
;;
*)
@@ -94,6 +97,10 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
TARGET_DIR="${CARGO_TARGET_DIR:-${REPO_ROOT}/target}"
mkdir -p "${TARGET_DIR}"
TARGET_DIR="$(cd "${TARGET_DIR}" && pwd)"
+if [[ -n "$RUNTIME_ARTIFACTS" && "$TARGET_DIR" != "$REPO_ROOT/target" ]]; then
+ echo "Runtime artifacts require CARGO_TARGET_DIR=$REPO_ROOT/target"
+ exit 1
+fi
export CARGO_TARGET_DIR="${TARGET_DIR}"
cd "${REPO_ROOT}"
@@ -263,14 +270,13 @@ else
rm -rf "${BASELINE_TARGET_DIR}"
fi
-# The baseline never writes here any more, but a binary left by an earlier run
-# of this script (or by any other build in this tree) would satisfy the
-# existence check below without cargo having produced it now. Delete it so that
-# check means what it says.
-rm -f "${HEAD_SERVER}"
-
-echo "Building HEAD iggy-server from ${HEAD_SHA}..."
-cargo build --locked -p server --bin iggy-server
+if [[ -n "$RUNTIME_ARTIFACTS" ]]; then
+ ./scripts/ci/restore-runtime.sh "$RUNTIME_ARTIFACTS"
+else
+ rm -f "${HEAD_SERVER}"
+ echo "Building HEAD iggy-server from ${HEAD_SHA}..."
+ cargo build --locked -p server --bin iggy-server
+fi
if [ ! -x "${HEAD_SERVER}" ]; then
echo "HEAD build did not produce ${HEAD_SERVER}"
diff --git a/scripts/ci/test-runtime-artifacts.py b/scripts/ci/test-runtime-artifacts.py
new file mode 100644
index 000000000..689f0620c
--- /dev/null
+++ b/scripts/ci/test-runtime-artifacts.py
@@ -0,0 +1,256 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Exercise runtime archive transfer and CI selection without compiling the workspace."""
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+
+import yaml
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+RESTORE = REPO_ROOT / "scripts/ci/restore-runtime.sh"
+SERVER_START = "./.github/actions/utils/server-start"
+DOWNLOAD_RUNTIME = "./.github/actions/utils/download-runtime"
+
+
+def run(*args, cwd=REPO_ROOT, env=None):
+ return subprocess.run(args, cwd=cwd, env=env, text=True, capture_output=True)
+
+
+class RuntimeArtifacts(unittest.TestCase):
+ def setUp(self):
+ self.temp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temp.cleanup)
+ self.root = Path(self.temp.name)
+ self.producer = self.root / "producer"
+ self.consumer = self.root / "consumer"
+ self.artifact = self.root / "artifact"
+ for directory in (self.producer / "target/debug", self.consumer, self.artifact):
+ directory.mkdir(parents=True)
+ self.env = os.environ.copy()
+ self.env["GIT_DIR"] = run("git", "rev-parse", "--absolute-git-dir").stdout.strip()
+ self.revision = run("git", "rev-parse", "HEAD").stdout.strip()
+ self.files = ["target/debug/iggy-server", "target/debug/iggy"]
+ for name in self.files:
+ binary = self.producer / name
+ binary.write_text("#!/bin/sh\nprintf 'shared-runtime\\n'\n")
+ binary.chmod(0o755)
+ self.pack()
+
+ def pack(self):
+ (self.artifact / "revision").write_text(self.revision + "\n")
+ result = run("tar", "--zstd", "-cf", str(self.artifact / "runtime.tar.zst"),
+ *self.files, cwd=self.producer)
+ self.assertEqual(result.returncode, 0, result.stderr)
+
+ def restore(self):
+ return run("bash", str(RESTORE), str(self.artifact), cwd=self.consumer, env=self.env)
+
+ def test_restore_preserves_executables_and_unrelated_files(self):
+ unrelated = self.consumer / "keep.txt"
+ unrelated.write_text("keep")
+ result = self.restore()
+ self.assertEqual(result.returncode, 0, result.stderr)
+ for name in self.files:
+ binary = self.consumer / name
+ self.assertTrue(os.access(binary, os.X_OK), name)
+ self.assertEqual(binary.read_bytes(), (self.producer / name).read_bytes())
+ execution = run(str(binary), cwd=self.consumer)
+ self.assertEqual(execution.returncode, 0, execution.stderr)
+ self.assertEqual(execution.stdout, "shared-runtime\n")
+ self.assertEqual(unrelated.read_text(), "keep")
+
+ def test_wrong_revision_fails_before_overwriting_existing_binary(self):
+ binary = self.consumer / self.files[0]
+ binary.parent.mkdir(parents=True)
+ binary.write_text("existing binary")
+ (self.artifact / "revision").write_text("0" * 40 + "\n")
+ result = self.restore()
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("does not match checkout", result.stdout)
+ self.assertEqual(binary.read_text(), "existing binary")
+
+ def test_corrupted_archive_is_rejected(self):
+ (self.artifact / "runtime.tar.zst").write_bytes(b"invalid archive")
+ result = self.restore()
+ self.assertNotEqual(result.returncode, 0)
+ self.assertFalse((self.consumer / self.files[0]).exists())
+
+ def test_missing_cli_is_not_accepted_as_a_complete_runtime(self):
+ self.files.pop()
+ self.pack()
+ result = self.restore()
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Runtime binary missing or not executable: target/debug/iggy", result.stdout)
+
+ def test_nonexecutable_server_is_rejected(self):
+ (self.producer / self.files[0]).chmod(0o644)
+ self.pack()
+ result = self.restore()
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Runtime binary missing or not executable: target/debug/iggy-server", result.stdout)
+
+ def test_rerun_restores_the_original_binary(self):
+ self.assertEqual(self.restore().returncode, 0)
+ (self.consumer / self.files[0]).write_text("overwritten by another build")
+ self.assertEqual(self.restore().returncode, 0)
+ self.assertEqual((self.consumer / self.files[0]).read_bytes(),
+ (self.producer / self.files[0]).read_bytes())
+
+
+class RuntimeBuildSelection(unittest.TestCase):
+ def setUp(self):
+ self.temp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temp.cleanup)
+ self.root = Path(self.temp.name)
+ action = yaml.safe_load((REPO_ROOT / ".github/actions/rust/pre-merge/action.yml").read_text())
+ self.script = next(step["run"] for step in action["runs"]["steps"]
+ if step.get("name") == "Build coverage artifacts")
+ self.revision = run("git", "rev-parse", "HEAD").stdout.strip()
+ self.env = dict(os.environ, RUNNER_TEMP=str(self.root), ARTIFACT_TASK="test-build",
+ TEST_SELECTION="affected", PLAN_STATUS="failure", PLAN_FILE="",
+ GITHUB_WORKSPACE=str(self.root), PATH=f"{self.root}:{os.environ['PATH']}")
+ git = self.root / "git"
+ git.write_text(f"#!/bin/sh\ncase $1 in\n rev-parse) echo {self.revision} ;;\n cat-file) echo 'parent fixture' ;;\n *) exit 1 ;;\nesac\n")
+ git.chmod(0o755)
+ cargo = self.root / "cargo"
+ cargo.write_text("""#!/usr/bin/env python3
+import json
+import os
+from pathlib import Path
+import sys
+with (Path(os.environ['RUNNER_TEMP']) / 'cargo-calls').open('a') as calls:
+ calls.write(json.dumps(sys.argv[1:]) + '\\n')
+if sys.argv[1:3] == ['nextest', 'archive']:
+ Path(sys.argv[sys.argv.index('--archive-file') + 1]).touch()
+""")
+ cargo.chmod(0o755)
+ generator = self.root / "gateways/kafka/scripts/ci-wire-fixtures.sh"
+ generator.parent.mkdir(parents=True)
+ generator.write_text("#!/bin/sh\nmkdir -p gateways/kafka/tools/kafka-tool/kafka_messages\n")
+ generator.chmod(0o755)
+
+ def assert_workspace_fallback(self):
+ result = run("bash", "-euo", "pipefail", "-c", self.script, cwd=self.root, env=self.env)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ artifact = self.root / "rust-test-build"
+ self.assertEqual((artifact / "scope").read_text(), "workspace\n")
+ self.assertEqual((artifact / "revision").read_text().strip(), self.revision)
+ calls = [json.loads(line) for line in (self.root / "cargo-calls").read_text().splitlines()]
+ self.assertEqual(calls[0], ["build", "--locked", "--workspace", "--exclude", "server",
+ "--exclude", "iggy-cli", "--message-format=json-render-diagnostics"])
+ self.assertEqual(calls[1], ["nextest", "archive", "--locked", "--profile", "ci",
+ "--archive-file", str(artifact / "nextest.tar.zst")])
+ self.assertTrue((artifact / "runtime.tar.zst").is_file())
+ self.assertEqual((artifact / "kafka-fixtures").read_text(), "true\n")
+
+ def test_planner_failure_builds_all_test_extras_and_tests(self):
+ self.assert_workspace_fallback()
+
+ def test_explicit_workspace_selection_does_not_need_a_planner(self):
+ self.env["TEST_SELECTION"] = "workspace"
+ self.assert_workspace_fallback()
+
+ def test_selector_failure_discards_partial_package_selection(self):
+ selector = self.root / "cargo-rail-action"
+ selector.write_text("""#!/usr/bin/env bash
+case "$2:$4" in
+ cargo-scope:*) printf 'packages\\n' ;;
+ cargo-args:*) printf '%s\\0' '-p' 'selected-test-package' ;;
+ target-args:*) printf '%s\\0' '--lib' ;;
+ package-names:cargo.test) printf '%s\\0' 'selected-test-package' ;;
+ *) exit 1 ;;
+esac
+""")
+ selector.chmod(0o755)
+ self.env["PLAN_STATUS"] = "success"
+ self.assert_workspace_fallback()
+
+
+class RuntimeSelection(unittest.TestCase):
+ def select(self, files):
+ workflow = yaml.safe_load((REPO_ROOT / ".github/workflows/_detect.yml").read_text())
+ components = yaml.safe_load((REPO_ROOT / ".github/config/components.yml").read_text())
+ step = next(step for step in workflow["jobs"]["detect"]["steps"] if step.get("id") == "mk")
+ script = step["with"]["script"]
+ script = script.replace("${{ steps.config.outputs.components }}", json.dumps(components))
+ script = script.replace("${{ steps.changed.outputs.all_files || '[]' }}", json.dumps(files))
+ with tempfile.TemporaryDirectory() as directory:
+ output = Path(directory) / "outputs"
+ env = dict(os.environ, GITHUB_OUTPUT=str(output))
+ result = run("node", "-e", "const context = {eventName: 'pull_request'};\n" + script, env=env)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ return dict(line.split("=", 1) for line in output.read_text().splitlines())
+
+ def test_component_config_change_runs_selection_validation(self):
+ outputs = self.select([".github/config/components.yml"])
+ self.assertIn(("ci-workflows", "validate"),
+ [(entry["component"], entry["task"])
+ for entry in json.loads(outputs["other_matrix"])["include"]])
+
+ def test_runtime_restore_change_exercises_real_consumers(self):
+ outputs = self.select(["scripts/ci/restore-runtime.sh"])
+ self.assertEqual(outputs["runtime"], "true")
+ self.assertTrue(json.loads(outputs["rust_runtime_plan"])["tests"])
+ self.assertIn(("ci-workflows", "validate"),
+ [(entry["component"], entry["task"])
+ for entry in json.loads(outputs["other_matrix"])["include"]])
+
+ def test_python_only_change_builds_runtime_without_rust_tests(self):
+ outputs = self.select(["foreign/python/src/lib.rs"])
+ self.assertEqual(outputs["runtime"], "true")
+ self.assertEqual(outputs["rust_plan"], "")
+ self.assertEqual(outputs["rust_runtime_plan"], "")
+ self.assertEqual(json.loads(outputs["python_matrix"])["include"][0]["component"], "sdk-python")
+
+ def test_server_change_selects_runtime_and_rust_tests(self):
+ outputs = self.select(["core/server/src/main.rs"])
+ self.assertEqual(outputs["runtime"], "true")
+ runtime_plan = json.loads(outputs["rust_runtime_plan"])
+ self.assertTrue(runtime_plan["tests"])
+ self.assertEqual([entry["task"] for entry in runtime_plan["checks"]], ["test-storage-compat"])
+ plan = json.loads(outputs["rust_plan"])
+ self.assertFalse(plan["tests"])
+ self.assertFalse({"test", "test-storage-compat"} & {entry["task"] for entry in plan["checks"]})
+
+ def test_swift_only_change_builds_server_for_e2e(self):
+ self.assertEqual(self.select(["foreign/swift/Sources/Iggy/IggyClient.swift"])["runtime"], "true")
+
+ def test_unrelated_documentation_does_not_build_server(self):
+ self.assertEqual(self.select(["docs/unrelated.md"])["runtime"], "false")
+
+
+class RuntimeConsumers(unittest.TestCase):
+ def test_every_server_start_follows_a_runtime_download(self):
+ for path in sorted((REPO_ROOT / ".github").rglob("*.yml")):
+ document = yaml.safe_load(path.read_text())
+ step_lists = [document.get("runs", {}).get("steps", [])]
+ step_lists += [job.get("steps", []) for job in document.get("jobs", {}).values()]
+ for steps in step_lists:
+ uses = [step.get("uses") for step in steps]
+ if SERVER_START in uses:
+ self.assertIn(DOWNLOAD_RUNTIME, uses[:uses.index(SERVER_START)], path.relative_to(REPO_ROOT))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/run-examples-from-readme.sh b/scripts/run-examples-from-readme.sh
index 3a13fb005..9e9ce521f 100755
--- a/scripts/run-examples-from-readme.sh
+++ b/scripts/run-examples-from-readme.sh
@@ -182,18 +182,19 @@ run_rust_examples() {
resolve_server_binary "${TARGET}" "iggy-server"
resolve_cli_binary "${TARGET}"
- # The README documents credentials as <iggy_username>/<iggy_password>
- # placeholders; the test server starts with iggy/iggy. The README keeps
- # plain cargo run commands, so the cross-compile target is injected here.
- if [ -n "${TARGET}" ]; then
- TRANSFORM_COMMAND() {
- echo "$1" | sed "s|<iggy_username>|iggy|g; s|<iggy_password>|iggy|g; s|cargo run |cargo run --target ${TARGET} |g"
- }
- else
- TRANSFORM_COMMAND() {
- echo "$1" | sed "s|<iggy_username>|iggy|g; s|<iggy_password>|iggy|g"
- }
- fi
+ TRANSFORM_COMMAND() {
+ local command="$1"
+ command="${command//<iggy_username>/iggy}"
+ command="${command//<iggy_password>/iggy}"
+ # Exercise the documented CLI arguments against the same binary as the other suites.
+ if [[ "$command" == 'cargo run --bin iggy -- '* ]]; then
+ printf '"%s" %s\n' "$CLI_BIN" "${command#cargo run --bin iggy -- }"
+ elif [ -n "${TARGET}" ]; then
+ printf '%s\n' "${command/cargo run /cargo run --target ${TARGET} }"
+ else
+ printf '%s\n' "$command"
+ fi
+ }
# Pre-flight: run CLI commands from root README
_rust_preflight() {