You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/22 15:22:56 UTC
(datafusion-comet) 01/01: ci: focus Miri on unsafe row and hash tests (#6072)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6072-0e60a0c79d5c20881deee51e054e1565cece6212
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit b2d06787824baeb08ebb2893678f7710615488db
Author: KUAN-HAO HUANG <10...@users.noreply.github.com>
AuthorDate: Tue Sep 22 14:54:57 2026 +0000
ci: focus Miri on unsafe row and hash tests (#6072)
* ci: focus Miri on unsafe row and hash tests
* ci: restrict Miri workflow token permissions
---
.github/workflows/miri.yml | 65 ++++++++++++++++++++++--
docs/source/contributor-guide/ci.md | 38 +++++++++++---
docs/source/contributor-guide/release_process.md | 4 +-
native/spark-expr/src/hash_funcs/murmur3.rs | 10 ++--
4 files changed, 98 insertions(+), 19 deletions(-)
diff --git a/.github/workflows/miri.yml b/.github/workflows/miri.yml
index f68e0b6d94..5310af2ee4 100644
--- a/.github/workflows/miri.yml
+++ b/.github/workflows/miri.yml
@@ -29,10 +29,26 @@ on:
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
workflow_dispatch:
+permissions:
+ contents: read
+
jobs:
miri:
- name: "Miri"
+ name: Miri (${{ matrix.suite }})
runs-on: ubuntu-24.04
+ strategy:
+ fail-fast: false
+ # Keep Miri focused on Comet's memory-access code. Whole-workspace tests
+ # abort in external FFI or platform-specific dependencies before these
+ # safety checks run.
+ matrix:
+ include:
+ - suite: Spark unsafe rows
+ package: datafusion-comet-shuffle
+ filter: 'spark_unsafe::'
+ - suite: Hash kernels
+ package: datafusion-comet-spark-expr
+ filter: 'hash_funcs::'
steps:
- uses: actions/checkout@v7
- name: Install Build Dependencies
@@ -43,10 +59,49 @@ jobs:
sudo apt-get install -y clang
- name: Install Miri
run: |
- rustup toolchain install nightly --component miri
- rustup override set nightly
- cargo miri setup
+ rustup toolchain install nightly --profile minimal --component miri
+ cargo +nightly miri setup
- name: Test with Miri
+ env:
+ PACKAGE: ${{ matrix.package }}
+ TEST_FILTER: ${{ matrix.filter }}
+ MIRIFLAGS: '-Zmiri-disable-isolation'
+ shell: bash
run: |
cd native
- MIRIFLAGS="-Zmiri-disable-isolation" cargo miri test --lib --bins --tests --examples
+ cargo +nightly miri test --locked -p "$PACKAGE" --lib "$TEST_FILTER" \
+ | tee "$RUNNER_TEMP/miri-tests.log"
+ # A renamed module or an all-ignored suite must not silently turn green.
+ if ! grep -Eq '^test result: ok\. [1-9][0-9]* passed;' "$RUNNER_TEMP/miri-tests.log"; then
+ echo "::error::Miri did not execute any tests for $PACKAGE / $TEST_FILTER"
+ exit 1
+ fi
+
+ nightly_report:
+ name: Nightly failure report
+ needs: miri
+ if: failure() && github.event_name == 'schedule'
+ permissions:
+ issues: write
+ runs-on: ubuntu-slim
+ steps:
+ - uses: actions/github-script@v9
+ with:
+ script: |
+ const label = 'ci-nightly-failure';
+ const { owner, repo } = context.repo;
+ const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`;
+ const day = new Date().toISOString().slice(0, 10);
+ const body = `The nightly Miri safety checks at ${context.sha.slice(0, 10)} failed: ${runUrl}`;
+
+ // Share the nightly CI issue so repeated failures do not open an issue every day.
+ const { data: open } = await github.rest.issues.listForRepo({
+ owner, repo, labels: label, state: 'open', per_page: 1,
+ });
+ if (open.length > 0) {
+ await github.rest.issues.createComment({ owner, repo, issue_number: open[0].number, body });
+ } else {
+ await github.rest.issues.create({
+ owner, repo, title: `Nightly CI failed on ${day}`, body, labels: [label],
+ });
+ }
diff --git a/docs/source/contributor-guide/ci.md b/docs/source/contributor-guide/ci.md
index 103b89f3c1..cea032370e 100644
--- a/docs/source/contributor-guide/ci.md
+++ b/docs/source/contributor-guide/ci.md
@@ -264,16 +264,40 @@ stays in one place. When you pick up a nightly failure:
Dispatching `ci.yml` from the Actions page with **Run workflow** runs every tier, including the
nightly suites, if you need a result before the next scheduled run.
+## Miri safety checks
+
+`miri.yml` runs nightly and supports manual dispatch. It runs the shuffle crate's
+`spark_unsafe::` tests and the expression crate's `hash_funcs::` tests in separate jobs,
+so a failure in one does not cancel the other. These cover Comet's unsafe row decoding
+and hash kernels. The job also fails if its filter selects no passing tests. A failed
+scheduled run opens or updates the same `ci-nightly-failure` issue used by nightly CI.
+
+Run either suite locally after installing nightly Rust with the Miri component:
+
+```sh
+cd native
+cargo +nightly miri setup
+MIRIFLAGS="-Zmiri-disable-isolation" cargo +nightly miri test --locked \
+ -p datafusion-comet-shuffle --lib 'spark_unsafe::'
+MIRIFLAGS="-Zmiri-disable-isolation" cargo +nightly miri test --locked \
+ -p datafusion-comet-spark-expr --lib 'hash_funcs::'
+```
+
+Miri cannot execute the JVM, cloud clients, or compression libraries through FFI.
+Running the whole workspace reaches those dependencies or dependency errors in
+platform synchronization before completing the safety checks. Regular Rust CI still runs the
+full tests. When adding unsafe code, add suitable tests to the Miri matrix as well;
+these focused suites do not cover every unsafe operation in Comet.
+
## Checking that the scheduled runs are healthy
-A scheduled run has no pull request to turn red, so when one breaks, nothing puts it in front of
-anyone. Comet has three daily schedules plus a weekly one, and two of the daily ones report nothing
-when they fail:
+A scheduled run has no pull request to turn red, so check both whether it ran and whether it
+reported a failure. Comet has three daily schedules plus a weekly one:
| Workflow | Cron (UTC) | Reports a failure? |
| ---------------------- | ------------ | ------------------------------------- |
| `publish_snapshot.yml` | `0 3 * * *` | no |
-| `miri.yml` | `0 4 * * *` | no |
+| `miri.yml` | `0 4 * * *` | yes |
| `ci.yml` nightly tier | `0 6 * * *` | yes, a `ci-nightly-failure` issue |
| `codeql.yml` | `16 4 * * 1` | yes, to the repository's Security tab |
@@ -293,9 +317,9 @@ Read the output for two different things:
with no activity for 60 days, and it does not announce either. Confirm the workflow is still
enabled with `gh api repos/apache/datafusion-comet/actions/workflows --jq '.workflows[] | "\(.state)\t\(.path)"'`,
and re-enable it from the Actions page if it is `disabled_inactivity`.
-- **A run of failures.** One red night is a flake or a real regression, and for `ci.yml` there is an
- issue open about it. Several consecutive red nights on `miri.yml` or `publish_snapshot.yml` means
- nobody has looked; treat the streak, not the newest run, as the thing to explain.
+- **A run of failures.** One red night is a flake or a real regression. `ci.yml` and `miri.yml`
+ open or update a failure issue; `publish_snapshot.yml` does not. Investigate repeated failures
+ even when a report already exists; treat the streak, not the newest run, as the thing to explain.
For the `ci.yml` nightly, green on its own does not mean the suites ran. Path filters and the diff
base are both allowed to select nothing — a documentation-only day legitimately runs no suite at
diff --git a/docs/source/contributor-guide/release_process.md b/docs/source/contributor-guide/release_process.md
index ae364b3692..e490cbec25 100644
--- a/docs/source/contributor-guide/release_process.md
+++ b/docs/source/contributor-guide/release_process.md
@@ -74,8 +74,8 @@ release to check for regressions. See the
Check that the scheduled CI runs have been healthy over the release window. Most of Comet's coverage of the
non-default Spark and Iceberg versions runs nightly rather than on pull requests, so a nightly run that has been
failing — or one that silently stopped firing — means the release is going out with less testing behind it than the
-tier table suggests. A scheduled run has no pull request to turn red, and `miri.yml` and `publish_snapshot.yml` do
-not report their own failures at all, so this has to be looked at deliberately. See
+tier table suggests. A scheduled run has no pull request to turn red, and `publish_snapshot.yml` does
+not report its own failures, so this has to be looked at deliberately. See
[Checking that the scheduled runs are healthy](ci.md#checking-that-the-scheduled-runs-are-healthy) for the commands
and for how to tell a genuinely quiet night from a broken one.
diff --git a/native/spark-expr/src/hash_funcs/murmur3.rs b/native/spark-expr/src/hash_funcs/murmur3.rs
index 5bf7399b19..29a4820ca5 100644
--- a/native/spark-expr/src/hash_funcs/murmur3.rs
+++ b/native/spark-expr/src/hash_funcs/murmur3.rs
@@ -833,7 +833,7 @@ mod tests {
use arrow::array::builder::{Int32Builder, ListBuilder, StringBuilder, StructBuilder};
use arrow::datatypes::{DataType, Field, Fields};
- fn build(payload_len: usize) -> ArrayRef {
+ fn build(data_capacity: usize) -> ArrayRef {
let fields: Fields = vec![
Arc::new(Field::new("a", DataType::Int32, true)),
Arc::new(Field::new("b", DataType::Utf8, true)),
@@ -843,11 +843,10 @@ mod tests {
fields,
vec![
Box::new(Int32Builder::new()),
- Box::new(StringBuilder::new()),
+ Box::new(StringBuilder::with_capacity(10, data_capacity)),
],
));
// Uneven lengths with an empty row and a null row, so rows drop out on different passes.
- let payload = "x".repeat(payload_len);
for (row, len) in [3usize, 0, 4, 1, 2].iter().enumerate() {
for i in 0..*len {
let sb = lb.values();
@@ -856,7 +855,7 @@ mod tests {
.append_value((row * 10 + i) as i32);
sb.field_builder::<StringBuilder>(1)
.unwrap()
- .append_value(&payload);
+ .append_value("xxxx");
sb.append(i % 3 != 2);
}
lb.append(row != 1);
@@ -875,7 +874,8 @@ mod tests {
"a small flat struct should be batched"
);
- // One wide value pushes the retained child past the limit, so the same shape is sliced.
+ // Retained capacity pushes the child past the limit without making Miri hash megabytes
+ // of padding. Both arrays contain the same values but take different paths.
let big = build(GATHER_ELIGIBLE_CHILD_BYTES);
let big_elements = big
.as_any()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org