You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/22 15:22:56 UTC

(datafusion-comet) 01/01: ci: focus Miri on unsafe row and hash tests (#6072)

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6072-0e60a0c79d5c20881deee51e054e1565cece6212
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git

commit b2d06787824baeb08ebb2893678f7710615488db
Author: KUAN-HAO HUANG <10...@users.noreply.github.com>
AuthorDate: Tue Sep 22 14:54:57 2026 +0000

    ci: focus Miri on unsafe row and hash tests (#6072)
    
    * ci: focus Miri on unsafe row and hash tests
    
    * ci: restrict Miri workflow token permissions
---
 .github/workflows/miri.yml                       | 65 ++++++++++++++++++++++--
 docs/source/contributor-guide/ci.md              | 38 +++++++++++---
 docs/source/contributor-guide/release_process.md |  4 +-
 native/spark-expr/src/hash_funcs/murmur3.rs      | 10 ++--
 4 files changed, 98 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/miri.yml b/.github/workflows/miri.yml
index f68e0b6d94..5310af2ee4 100644
--- a/.github/workflows/miri.yml
+++ b/.github/workflows/miri.yml
@@ -29,10 +29,26 @@ on:
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   miri:
-    name: "Miri"
+    name: Miri (${{ matrix.suite }})
     runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      # Keep Miri focused on Comet's memory-access code. Whole-workspace tests
+      # abort in external FFI or platform-specific dependencies before these
+      # safety checks run.
+      matrix:
+        include:
+          - suite: Spark unsafe rows
+            package: datafusion-comet-shuffle
+            filter: 'spark_unsafe::'
+          - suite: Hash kernels
+            package: datafusion-comet-spark-expr
+            filter: 'hash_funcs::'
     steps:
       - uses: actions/checkout@v7
       - name: Install Build Dependencies
@@ -43,10 +59,49 @@ jobs:
           sudo apt-get install -y clang
       - name: Install Miri
         run: |
-          rustup toolchain install nightly --component miri
-          rustup override set nightly
-          cargo miri setup
+          rustup toolchain install nightly --profile minimal --component miri
+          cargo +nightly miri setup
       - name: Test with Miri
+        env:
+          PACKAGE: ${{ matrix.package }}
+          TEST_FILTER: ${{ matrix.filter }}
+          MIRIFLAGS: '-Zmiri-disable-isolation'
+        shell: bash
         run: |
           cd native
-          MIRIFLAGS="-Zmiri-disable-isolation" cargo miri test --lib --bins --tests --examples
+          cargo +nightly miri test --locked -p "$PACKAGE" --lib "$TEST_FILTER" \
+            | tee "$RUNNER_TEMP/miri-tests.log"
+          # A renamed module or an all-ignored suite must not silently turn green.
+          if ! grep -Eq '^test result: ok\. [1-9][0-9]* passed;' "$RUNNER_TEMP/miri-tests.log"; then
+            echo "::error::Miri did not execute any tests for $PACKAGE / $TEST_FILTER"
+            exit 1
+          fi
+
+  nightly_report:
+    name: Nightly failure report
+    needs: miri
+    if: failure() && github.event_name == 'schedule'
+    permissions:
+      issues: write
+    runs-on: ubuntu-slim
+    steps:
+      - uses: actions/github-script@v9
+        with:
+          script: |
+            const label = 'ci-nightly-failure';
+            const { owner, repo } = context.repo;
+            const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`;
+            const day = new Date().toISOString().slice(0, 10);
+            const body = `The nightly Miri safety checks at ${context.sha.slice(0, 10)} failed: ${runUrl}`;
+
+            // Share the nightly CI issue so repeated failures do not open an issue every day.
+            const { data: open } = await github.rest.issues.listForRepo({
+              owner, repo, labels: label, state: 'open', per_page: 1,
+            });
+            if (open.length > 0) {
+              await github.rest.issues.createComment({ owner, repo, issue_number: open[0].number, body });
+            } else {
+              await github.rest.issues.create({
+                owner, repo, title: `Nightly CI failed on ${day}`, body, labels: [label],
+              });
+            }
diff --git a/docs/source/contributor-guide/ci.md b/docs/source/contributor-guide/ci.md
index 103b89f3c1..cea032370e 100644
--- a/docs/source/contributor-guide/ci.md
+++ b/docs/source/contributor-guide/ci.md
@@ -264,16 +264,40 @@ stays in one place. When you pick up a nightly failure:
 Dispatching `ci.yml` from the Actions page with **Run workflow** runs every tier, including the
 nightly suites, if you need a result before the next scheduled run.
 
+## Miri safety checks
+
+`miri.yml` runs nightly and supports manual dispatch. It runs the shuffle crate's
+`spark_unsafe::` tests and the expression crate's `hash_funcs::` tests in separate jobs,
+so a failure in one does not cancel the other. These cover Comet's unsafe row decoding
+and hash kernels. The job also fails if its filter selects no passing tests. A failed
+scheduled run opens or updates the same `ci-nightly-failure` issue used by nightly CI.
+
+Run either suite locally after installing nightly Rust with the Miri component:
+
+```sh
+cd native
+cargo +nightly miri setup
+MIRIFLAGS="-Zmiri-disable-isolation" cargo +nightly miri test --locked \
+  -p datafusion-comet-shuffle --lib 'spark_unsafe::'
+MIRIFLAGS="-Zmiri-disable-isolation" cargo +nightly miri test --locked \
+  -p datafusion-comet-spark-expr --lib 'hash_funcs::'
+```
+
+Miri cannot execute the JVM, cloud clients, or compression libraries through FFI.
+Running the whole workspace reaches those dependencies or dependency errors in
+platform synchronization before completing the safety checks. Regular Rust CI still runs the
+full tests. When adding unsafe code, add suitable tests to the Miri matrix as well;
+these focused suites do not cover every unsafe operation in Comet.
+
 ## Checking that the scheduled runs are healthy
 
-A scheduled run has no pull request to turn red, so when one breaks, nothing puts it in front of
-anyone. Comet has three daily schedules plus a weekly one, and two of the daily ones report nothing
-when they fail:
+A scheduled run has no pull request to turn red, so check both whether it ran and whether it
+reported a failure. Comet has three daily schedules plus a weekly one:
 
 | Workflow               | Cron (UTC)   | Reports a failure?                    |
 | ---------------------- | ------------ | ------------------------------------- |
 | `publish_snapshot.yml` | `0 3 * * *`  | no                                    |
-| `miri.yml`             | `0 4 * * *`  | no                                    |
+| `miri.yml`             | `0 4 * * *`  | yes                                   |
 | `ci.yml` nightly tier  | `0 6 * * *`  | yes, a `ci-nightly-failure` issue     |
 | `codeql.yml`           | `16 4 * * 1` | yes, to the repository's Security tab |
 
@@ -293,9 +317,9 @@ Read the output for two different things:
   with no activity for 60 days, and it does not announce either. Confirm the workflow is still
   enabled with `gh api repos/apache/datafusion-comet/actions/workflows --jq '.workflows[] | "\(.state)\t\(.path)"'`,
   and re-enable it from the Actions page if it is `disabled_inactivity`.
-- **A run of failures.** One red night is a flake or a real regression, and for `ci.yml` there is an
-  issue open about it. Several consecutive red nights on `miri.yml` or `publish_snapshot.yml` means
-  nobody has looked; treat the streak, not the newest run, as the thing to explain.
+- **A run of failures.** One red night is a flake or a real regression. `ci.yml` and `miri.yml`
+  open or update a failure issue; `publish_snapshot.yml` does not. Investigate repeated failures
+  even when a report already exists; treat the streak, not the newest run, as the thing to explain.
 
 For the `ci.yml` nightly, green on its own does not mean the suites ran. Path filters and the diff
 base are both allowed to select nothing — a documentation-only day legitimately runs no suite at
diff --git a/docs/source/contributor-guide/release_process.md b/docs/source/contributor-guide/release_process.md
index ae364b3692..e490cbec25 100644
--- a/docs/source/contributor-guide/release_process.md
+++ b/docs/source/contributor-guide/release_process.md
@@ -74,8 +74,8 @@ release to check for regressions. See the
 Check that the scheduled CI runs have been healthy over the release window. Most of Comet's coverage of the
 non-default Spark and Iceberg versions runs nightly rather than on pull requests, so a nightly run that has been
 failing — or one that silently stopped firing — means the release is going out with less testing behind it than the
-tier table suggests. A scheduled run has no pull request to turn red, and `miri.yml` and `publish_snapshot.yml` do
-not report their own failures at all, so this has to be looked at deliberately. See
+tier table suggests. A scheduled run has no pull request to turn red, and `publish_snapshot.yml` does
+not report its own failures, so this has to be looked at deliberately. See
 [Checking that the scheduled runs are healthy](ci.md#checking-that-the-scheduled-runs-are-healthy) for the commands
 and for how to tell a genuinely quiet night from a broken one.
 
diff --git a/native/spark-expr/src/hash_funcs/murmur3.rs b/native/spark-expr/src/hash_funcs/murmur3.rs
index 5bf7399b19..29a4820ca5 100644
--- a/native/spark-expr/src/hash_funcs/murmur3.rs
+++ b/native/spark-expr/src/hash_funcs/murmur3.rs
@@ -833,7 +833,7 @@ mod tests {
         use arrow::array::builder::{Int32Builder, ListBuilder, StringBuilder, StructBuilder};
         use arrow::datatypes::{DataType, Field, Fields};
 
-        fn build(payload_len: usize) -> ArrayRef {
+        fn build(data_capacity: usize) -> ArrayRef {
             let fields: Fields = vec![
                 Arc::new(Field::new("a", DataType::Int32, true)),
                 Arc::new(Field::new("b", DataType::Utf8, true)),
@@ -843,11 +843,10 @@ mod tests {
                 fields,
                 vec![
                     Box::new(Int32Builder::new()),
-                    Box::new(StringBuilder::new()),
+                    Box::new(StringBuilder::with_capacity(10, data_capacity)),
                 ],
             ));
             // Uneven lengths with an empty row and a null row, so rows drop out on different passes.
-            let payload = "x".repeat(payload_len);
             for (row, len) in [3usize, 0, 4, 1, 2].iter().enumerate() {
                 for i in 0..*len {
                     let sb = lb.values();
@@ -856,7 +855,7 @@ mod tests {
                         .append_value((row * 10 + i) as i32);
                     sb.field_builder::<StringBuilder>(1)
                         .unwrap()
-                        .append_value(&payload);
+                        .append_value("xxxx");
                     sb.append(i % 3 != 2);
                 }
                 lb.append(row != 1);
@@ -875,7 +874,8 @@ mod tests {
             "a small flat struct should be batched"
         );
 
-        // One wide value pushes the retained child past the limit, so the same shape is sliced.
+        // Retained capacity pushes the child past the limit without making Miri hash megabytes
+        // of padding. Both arrays contain the same values but take different paths.
         let big = build(GATHER_ELIGIBLE_CHILD_BYTES);
         let big_elements = big
             .as_any()


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org