You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/03/29 07:40:46 UTC
(datafusion) branch main updated: No cargo test for `sort_mem_validation` (#21222)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 1a0af76f78 No cargo test for `sort_mem_validation` (#21222)
1a0af76f78 is described below
commit 1a0af76f780708ff9d9e1940686748bd743352d3
Author: Dmitrii Blaginin <dm...@blaginin.me>
AuthorDate: Sun Mar 29 08:40:23 2026 +0100
No cargo test for `sort_mem_validation` (#21222)
Every time I run tests with `sort_mem_validation` i'm forced to
recompile those tests several times, as those trigger `cargo
test` (=recompilation with different features). I made it use the
existing compiled binary
Should save us 2-3 minutes on recompiling code on CI
---
.../memory_limit_validation/sort_mem_validation.rs | 54 +++-------------------
1 file changed, 7 insertions(+), 47 deletions(-)
diff --git a/datafusion/core/tests/memory_limit/memory_limit_validation/sort_mem_validation.rs b/datafusion/core/tests/memory_limit/memory_limit_validation/sort_mem_validation.rs
index e1d5f1b1ab..bf04123fff 100644
--- a/datafusion/core/tests/memory_limit/memory_limit_validation/sort_mem_validation.rs
+++ b/datafusion/core/tests/memory_limit/memory_limit_validation/sort_mem_validation.rs
@@ -21,14 +21,10 @@
//! This file is organized as:
//! - Test runners that spawn individual test processes
//! - Test cases that contain the actual validation logic
-use log::info;
-use std::sync::Once;
use std::{process::Command, str};
use crate::memory_limit::memory_limit_validation::utils;
-static INIT: Once = Once::new();
-
// ===========================================================================
// Test runners:
// Runners are split into multiple tests to run in parallel
@@ -69,49 +65,16 @@ fn sort_with_mem_limit_2_cols_2_runner() {
spawn_test_process("sort_with_mem_limit_2_cols_2");
}
-/// `spawn_test_process` might trigger multiple recompilations and the test binary
-/// size might grow indefinitely. This initializer ensures recompilation is only done
-/// once and the target size is bounded.
-///
-/// TODO: This is a hack, can be cleaned up if we have a better way to let multiple
-/// test cases run in different processes (instead of different threads by default)
-fn init_once() {
- INIT.call_once(|| {
- let _ = Command::new("cargo")
- .arg("test")
- .arg("--no-run")
- .arg("--package")
- .arg("datafusion")
- .arg("--test")
- .arg("core_integration")
- .arg("--features")
- .arg("extended_tests")
- .env("DATAFUSION_TEST_MEM_LIMIT_VALIDATION", "1")
- .output()
- .expect("Failed to execute test command");
- });
-}
-
-/// Helper function that executes a test in a separate process with the required environment
-/// variable set. Memory limit validation tasks need to measure memory resident set
-/// size (RSS), so they must run in a separate process.
+/// Helper function that executes a test in a separate process with the required
+/// environment variable set. Re-invokes the current test binary directly,
+/// avoiding cargo overhead and recompilation.
fn spawn_test_process(test: &str) {
- init_once();
-
let test_path =
format!("memory_limit::memory_limit_validation::sort_mem_validation::{test}");
- info!("Running test: {test_path}");
-
- // Run the test command
- let output = Command::new("cargo")
- .arg("test")
- .arg("--package")
- .arg("datafusion")
- .arg("--test")
- .arg("core_integration")
- .arg("--features")
- .arg("extended_tests")
- .arg("--")
+
+ let exe = std::env::current_exe().expect("Failed to get test binary path");
+
+ let output = Command::new(exe)
.arg(&test_path)
.arg("--exact")
.arg("--nocapture")
@@ -119,12 +82,9 @@ fn spawn_test_process(test: &str) {
.output()
.expect("Failed to execute test command");
- // Convert output to strings
let stdout = str::from_utf8(&output.stdout).unwrap_or("");
let stderr = str::from_utf8(&output.stderr).unwrap_or("");
- info!("{stdout}");
-
assert!(
output.status.success(),
"Test '{}' failed with status: {}\nstdout:\n{}\nstderr:\n{}",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org