You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/03/29 07:40:46 UTC

(datafusion) branch main updated: No cargo test for `sort_mem_validation` (#21222)

This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git


The following commit(s) were added to refs/heads/main by this push:
     new 1a0af76f78 No cargo test for `sort_mem_validation` (#21222)
1a0af76f78 is described below

commit 1a0af76f780708ff9d9e1940686748bd743352d3
Author: Dmitrii Blaginin <dm...@blaginin.me>
AuthorDate: Sun Mar 29 08:40:23 2026 +0100

    No cargo test for `sort_mem_validation` (#21222)
    
    Every time I run tests with `sort_mem_validation` i'm forced to
    recompile those tests several times, as those trigger `cargo
    test` (=recompilation with different features). I made it use the
    existing compiled binary
    
    Should save us 2-3 minutes on recompiling code on CI
---
 .../memory_limit_validation/sort_mem_validation.rs | 54 +++-------------------
 1 file changed, 7 insertions(+), 47 deletions(-)

diff --git a/datafusion/core/tests/memory_limit/memory_limit_validation/sort_mem_validation.rs b/datafusion/core/tests/memory_limit/memory_limit_validation/sort_mem_validation.rs
index e1d5f1b1ab..bf04123fff 100644
--- a/datafusion/core/tests/memory_limit/memory_limit_validation/sort_mem_validation.rs
+++ b/datafusion/core/tests/memory_limit/memory_limit_validation/sort_mem_validation.rs
@@ -21,14 +21,10 @@
 //! This file is organized as:
 //! - Test runners that spawn individual test processes
 //! - Test cases that contain the actual validation logic
-use log::info;
-use std::sync::Once;
 use std::{process::Command, str};
 
 use crate::memory_limit::memory_limit_validation::utils;
 
-static INIT: Once = Once::new();
-
 // ===========================================================================
 // Test runners:
 // Runners are split into multiple tests to run in parallel
@@ -69,49 +65,16 @@ fn sort_with_mem_limit_2_cols_2_runner() {
     spawn_test_process("sort_with_mem_limit_2_cols_2");
 }
 
-/// `spawn_test_process` might trigger multiple recompilations and the test binary
-/// size might grow indefinitely. This initializer ensures recompilation is only done
-/// once and the target size is bounded.
-///
-/// TODO: This is a hack, can be cleaned up if we have a better way to let multiple
-/// test cases run in different processes (instead of different threads by default)
-fn init_once() {
-    INIT.call_once(|| {
-        let _ = Command::new("cargo")
-            .arg("test")
-            .arg("--no-run")
-            .arg("--package")
-            .arg("datafusion")
-            .arg("--test")
-            .arg("core_integration")
-            .arg("--features")
-            .arg("extended_tests")
-            .env("DATAFUSION_TEST_MEM_LIMIT_VALIDATION", "1")
-            .output()
-            .expect("Failed to execute test command");
-    });
-}
-
-/// Helper function that executes a test in a separate process with the required environment
-/// variable set. Memory limit validation tasks need to measure memory resident set
-/// size (RSS), so they must run in a separate process.
+/// Helper function that executes a test in a separate process with the required
+/// environment variable set. Re-invokes the current test binary directly,
+/// avoiding cargo overhead and recompilation.
 fn spawn_test_process(test: &str) {
-    init_once();
-
     let test_path =
         format!("memory_limit::memory_limit_validation::sort_mem_validation::{test}");
-    info!("Running test: {test_path}");
-
-    // Run the test command
-    let output = Command::new("cargo")
-        .arg("test")
-        .arg("--package")
-        .arg("datafusion")
-        .arg("--test")
-        .arg("core_integration")
-        .arg("--features")
-        .arg("extended_tests")
-        .arg("--")
+
+    let exe = std::env::current_exe().expect("Failed to get test binary path");
+
+    let output = Command::new(exe)
         .arg(&test_path)
         .arg("--exact")
         .arg("--nocapture")
@@ -119,12 +82,9 @@ fn spawn_test_process(test: &str) {
         .output()
         .expect("Failed to execute test command");
 
-    // Convert output to strings
     let stdout = str::from_utf8(&output.stdout).unwrap_or("");
     let stderr = str::from_utf8(&output.stderr).unwrap_or("");
 
-    info!("{stdout}");
-
     assert!(
         output.status.success(),
         "Test '{}' failed with status: {}\nstdout:\n{}\nstderr:\n{}",


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org