You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by mi...@apache.org on 2026/10/15 15:37:02 UTC
(datafusion-ballista) branch main updated: fix(executor): drain Flight before shutdown cleanup (#2448)
This is an automated email from the ASF dual-hosted git repository.
milenkovicm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new d15184476 fix(executor): drain Flight before shutdown cleanup (#2448)
d15184476 is described below
commit d151844766ea034902b14bc20ff312fc3a77871b
Author: Dan <da...@gmail.com>
AuthorDate: Thu Oct 15 16:36:57 2026 +0100
fix(executor): drain Flight before shutdown cleanup (#2448)
---
Cargo.lock | 1 +
ballista/executor/src/config.rs | 4 +
ballista/executor/src/executor_process.rs | 144 +++++++++++++++++++-----------
chaos-testing/Cargo.toml | 1 +
chaos-testing/src/bin/chaos-executor.rs | 10 +++
chaos-testing/src/cluster.rs | 41 ++++++++-
chaos-testing/src/fixture.rs | 7 ++
chaos-testing/tests/ha.rs | 65 ++++++++++++--
8 files changed, 212 insertions(+), 61 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 562607cb6..902aef808 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1069,6 +1069,7 @@ dependencies = [
"ballista-scheduler",
"datafusion",
"env_logger",
+ "futures",
"log",
"nix 0.31.3",
"reqwest 0.13.5",
diff --git a/ballista/executor/src/config.rs b/ballista/executor/src/config.rs
index a812f487c..d8965a5e3 100644
--- a/ballista/executor/src/config.rs
+++ b/ballista/executor/src/config.rs
@@ -104,6 +104,9 @@ pub struct Config {
help = "How long to try connecting to scheduler before failing. Set to zero to fail after first attempt."
)]
pub scheduler_connect_timeout_seconds: u16,
+ /// Maximum time in seconds for executor graceful shutdown (0 = wait indefinitely).
+ #[arg(long, default_value_t = 10)]
+ pub graceful_shutdown_timeout_seconds: u64,
/// Directory for storing temporary shuffle data and IPC files.
#[arg(long, help = "Directory for temporary IPC files")]
pub work_dir: Option<String>,
@@ -264,6 +267,7 @@ impl TryFrom<Config> for ExecutorProcessConfig {
scheduler_host: opt.scheduler_host,
scheduler_port: opt.scheduler_port,
scheduler_connect_timeout_seconds: opt.scheduler_connect_timeout_seconds,
+ graceful_shutdown_timeout_seconds: opt.graceful_shutdown_timeout_seconds,
vcores,
task_scheduling_policy: opt.task_scheduling_policy,
work_dir: opt.work_dir,
diff --git a/ballista/executor/src/executor_process.rs b/ballista/executor/src/executor_process.rs
index 1f72c925f..1e64220e0 100644
--- a/ballista/executor/src/executor_process.rs
+++ b/ballista/executor/src/executor_process.rs
@@ -298,6 +298,9 @@ pub struct ExecutorProcessConfig {
pub scheduler_port: u16,
/// Timeout in seconds for establishing scheduler connection.
pub scheduler_connect_timeout_seconds: u16,
+ /// Maximum time in seconds for scheduler notification, task and Flight draining,
+ /// and shuffle cleanup after shutdown is requested. `0` waits indefinitely.
+ pub graceful_shutdown_timeout_seconds: u64,
/// Virtual cores advertised by this executor to the scheduler.
pub vcores: usize,
/// Task scheduling policy (pull-staged or push-staged).
@@ -403,6 +406,7 @@ impl Default for ExecutorProcessConfig {
scheduler_host: "localhost".into(),
scheduler_port: 50050,
scheduler_connect_timeout_seconds: 0,
+ graceful_shutdown_timeout_seconds: 10,
vcores: std::thread::available_parallelism().unwrap().get(),
task_scheduling_policy: Default::default(),
log_dir: None,
@@ -727,10 +731,12 @@ async fn start_executor_process_inner(
// Graceful shutdown notification
let shutdown_notification = ShutdownNotifier::new();
let flight_work_dir = work_dir.clone();
+ let clean_job_data_on_shutdown = opt.job_data_clean_up_interval_seconds > 0;
- if opt.job_data_clean_up_interval_seconds > 0 {
+ if clean_job_data_on_shutdown {
let mut interval_time =
time::interval(Duration::from_secs(opt.job_data_clean_up_interval_seconds));
+ let cleaner_work_dir = work_dir.clone();
let mut shuffle_cleaner_shutdown = shutdown_notification.subscribe_for_shutdown();
let shuffle_cleaner_complete = shutdown_notification.shutdown_complete_tx.clone();
@@ -740,18 +746,12 @@ async fn start_executor_process_inner(
while !shuffle_cleaner_shutdown.is_shutdown() {
tokio::select! {
_ = interval_time.tick() => {
- if let Err(e) = clean_shuffle_data_loop(&work_dir, job_data_ttl_seconds).await
+ if let Err(e) = clean_shuffle_data_loop(&cleaner_work_dir, job_data_ttl_seconds).await
{
error!("Ballista executor fail to clean_shuffle_data {e:?}")
}
},
_ = shuffle_cleaner_shutdown.recv() => {
- if let Err(e) = clean_all_shuffle_data(&work_dir).await
- {
- error!("Ballista executor fail to clean_shuffle_data {e:?}")
- } else {
- info!("Shuffle data cleaned.");
- }
drop(shuffle_cleaner_complete);
return;
}
@@ -793,14 +793,21 @@ async fn start_executor_process_inner(
);
}
_ => {
- service_handlers.push(tokio::spawn(execution_loop::poll_loop(
+ let poll_loop = execution_loop::poll_loop(
scheduler.clone(),
executor.clone(),
default_codec,
None, // poll_now_notify
None, // free_vcores: use internal semaphore
health,
- )));
+ );
+ let mut poll_shutdown = shutdown_notification.subscribe_for_shutdown();
+ service_handlers.push(tokio::spawn(async move {
+ tokio::select! {
+ result = poll_loop => result,
+ _ = poll_shutdown.recv() => Ok(()),
+ }
+ }));
}
};
let shutdown = shutdown_notification.subscribe_for_shutdown();
@@ -860,57 +867,86 @@ async fn start_executor_process_inner(
info!("setting executor to TERMINATING status");
TERMINATING.store(true, Ordering::Release);
- if notify_scheduler {
- // Send a heartbeat to update status of executor to `Fenced`. This should signal to the
- // scheduler to no longer schedule tasks on this executor
- if let Err(error) = scheduler
- .heart_beat_from_executor(HeartBeatParams {
- executor_id: executor_id.clone(),
- metrics: vec![],
- status: Some(ExecutorStatus {
- status: Some(Status::Terminating(String::default())),
- }),
- metadata: Some(executor_meta),
- })
- .await
- {
- error!("error sending heartbeat with fenced status: {error:?}");
- }
-
- // TODO we probably don't need a separate rpc call for this....
- if let Err(error) = scheduler
- .executor_stopped(ExecutorStoppedParams {
- executor_id,
- reason: stop_reason,
- })
- .await
- {
- error!("ExecutorStopped grpc failed: {error:?}");
- }
-
- // Wait for tasks to drain
- tasks_drained.await;
- }
-
- // Extract the `shutdown_complete` receiver and transmitter
- // explicitly drop `shutdown_transmitter`. This is important, as the
- // `.await` below would otherwise never complete.
let ShutdownNotifier {
mut shutdown_complete_rx,
shutdown_complete_tx,
notify_shutdown,
- ..
} = shutdown_notification;
- // When `notify_shutdown` is dropped, all components which have `subscribe`d will
- // receive the shutdown signal and can exit
- let _ = notify_shutdown.send(());
- drop(notify_shutdown);
- // Drop final `Sender` so the `Receiver` below can complete
- drop(shutdown_complete_tx);
+ let graceful_shutdown = async {
+ if notify_scheduler {
+ // Send a heartbeat to update status of executor to `Fenced`. This should signal to the
+ // scheduler to no longer schedule tasks on this executor
+ if let Err(error) = scheduler
+ .heart_beat_from_executor(HeartBeatParams {
+ executor_id: executor_id.clone(),
+ metrics: vec![],
+ status: Some(ExecutorStatus {
+ status: Some(Status::Terminating(String::default())),
+ }),
+ metadata: Some(executor_meta),
+ })
+ .await
+ {
+ error!("error sending heartbeat with fenced status: {error:?}");
+ }
+
+ // TODO we probably don't need a separate rpc call for this....
+ if let Err(error) = scheduler
+ .executor_stopped(ExecutorStoppedParams {
+ executor_id,
+ reason: stop_reason,
+ })
+ .await
+ {
+ error!("ExecutorStopped grpc failed: {error:?}");
+ }
+
+ // Wait for tasks to drain
+ tasks_drained.await;
+ }
- // Wait for all related components to finish the shutdown processing.
- let _ = shutdown_complete_rx.recv().await;
+ // Signal all subscribed components to shut down.
+ let _ = notify_shutdown.send(());
+ // Drop final `Sender` so the `Receiver` below can complete
+ drop(shutdown_complete_tx);
+
+ // Wait for all related components to finish the shutdown processing.
+ let _ = shutdown_complete_rx.recv().await;
+ // Drain the servers, including Flight, before removing their shuffle data.
+ while !service_handlers.is_empty() {
+ if let Err(error) = check_services(&mut service_handlers).await {
+ error!("Executor service failed during shutdown: {error:?}");
+ }
+ }
+ if clean_job_data_on_shutdown {
+ if let Err(e) = clean_all_shuffle_data(&work_dir).await {
+ error!("Ballista executor fail to clean_shuffle_data {e:?}")
+ } else {
+ info!("Shuffle data cleaned.");
+ }
+ }
+ };
+
+ let shutdown_timeout = Duration::from_secs(opt.graceful_shutdown_timeout_seconds);
+ if shutdown_timeout.is_zero() {
+ graceful_shutdown.await;
+ } else if time::timeout(shutdown_timeout, graceful_shutdown)
+ .await
+ .is_err()
+ {
+ let message = format!(
+ "Executor graceful shutdown timed out after {} seconds",
+ shutdown_timeout.as_secs()
+ );
+ warn!("{message}; aborting remaining services");
+ // A scheduler RPC or task drain may have timed out before broadcasting shutdown.
+ let _ = notify_shutdown.send(());
+ for service in &service_handlers {
+ service.abort();
+ }
+ return Err(BallistaError::General(message));
+ }
info!("Executor stopped.");
Ok(())
}
diff --git a/chaos-testing/Cargo.toml b/chaos-testing/Cargo.toml
index 477adbc70..2100d64bf 100644
--- a/chaos-testing/Cargo.toml
+++ b/chaos-testing/Cargo.toml
@@ -47,6 +47,7 @@ tempfile = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "sync", "time", "net", "signal"] }
[dev-dependencies]
+futures = { workspace = true }
rstest = { workspace = true }
[lib]
diff --git a/chaos-testing/src/bin/chaos-executor.rs b/chaos-testing/src/bin/chaos-executor.rs
index f8d3a765d..48346fcbe 100644
--- a/chaos-testing/src/bin/chaos-executor.rs
+++ b/chaos-testing/src/bin/chaos-executor.rs
@@ -52,6 +52,16 @@ async fn main() -> ballista_core::error::Result<()> {
.unwrap_or_else(|_| "127.0.0.1".into()),
scheduler_port: env_u16("CHAOS_SCHEDULER_PORT"),
scheduler_connect_timeout_seconds: 10,
+ graceful_shutdown_timeout_seconds: std::env::var(
+ "CHAOS_GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS",
+ )
+ .ok()
+ .map(|value| {
+ value
+ .parse()
+ .expect("Graceful shutdown timeout must be a u64")
+ })
+ .unwrap_or(10),
vcores: std::env::var("CHAOS_CONCURRENT_TASKS")
.ok()
.and_then(|v| v.parse().ok())
diff --git a/chaos-testing/src/cluster.rs b/chaos-testing/src/cluster.rs
index 5f03252cc..1623d80fa 100644
--- a/chaos-testing/src/cluster.rs
+++ b/chaos-testing/src/cluster.rs
@@ -20,7 +20,7 @@ use nix::fcntl::{Flock, FlockArg};
use std::fs::{File, OpenOptions};
use std::net::TcpListener;
use std::path::{Path, PathBuf};
-use std::process::{Child, Command, Stdio};
+use std::process::{Child, Command, ExitStatus, Stdio};
use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant};
use tokio::sync::{Mutex, OwnedMutexGuard};
@@ -99,6 +99,7 @@ pub struct TestClusterBuilder {
stage_max_failures: usize,
concurrent_tasks: usize,
no_executors_grace_seconds: u64,
+ graceful_shutdown_timeout_seconds: u64,
result_fetch: Option<ResultFetch>,
}
@@ -116,6 +117,8 @@ impl Default for TestClusterBuilder {
// Ballista's default is 30s. A short grace makes the total-loss
// scenario fail the job a second or so after the reap.
no_executors_grace_seconds: 1,
+ // Let test timeouts bound shutdown so slow CI can finish draining.
+ graceful_shutdown_timeout_seconds: 0,
result_fetch: None,
}
}
@@ -427,6 +430,10 @@ impl TestCluster {
)
.env("CHAOS_WORK_DIR", work_dir.display().to_string())
.env("CHAOS_HEARTBEAT_SECONDS", "1")
+ .env(
+ "CHAOS_GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS",
+ self.builder.graceful_shutdown_timeout_seconds.to_string(),
+ )
.env(
"RUST_LOG",
std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into()),
@@ -725,6 +732,38 @@ impl TestCluster {
Ok(())
}
+ /// Request graceful shutdown without waiting, so a client can keep streaming.
+ pub fn terminate_executor(&self, index: usize) -> Result<(), String> {
+ use nix::sys::signal::{Signal, kill};
+ use nix::unistd::Pid;
+
+ let pid = self.executors[index].child.id();
+ kill(Pid::from_raw(pid as i32), Signal::SIGTERM).map_err(|e| e.to_string())
+ }
+
+ pub async fn await_executor_exit(
+ &mut self,
+ index: usize,
+ ) -> Result<ExitStatus, String> {
+ let deadline = Instant::now() + Duration::from_secs(30);
+ loop {
+ if let Some(status) = self.executors[index]
+ .child
+ .try_wait()
+ .map_err(|e| e.to_string())?
+ {
+ return Ok(status);
+ }
+ if Instant::now() > deadline {
+ return Err(format!(
+ "executor {index} did not exit\n{}",
+ self.log_tails()
+ ));
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ }
+
pub async fn await_successful_shuffle_output(
&self,
job_id: &str,
diff --git a/chaos-testing/src/fixture.rs b/chaos-testing/src/fixture.rs
index f8c6d8784..4fcbd2b61 100644
--- a/chaos-testing/src/fixture.rs
+++ b/chaos-testing/src/fixture.rs
@@ -32,6 +32,8 @@ const FACT_FILES: i64 = 8;
/// is what creates the multi-stage plan the HA paths need.
const DIM_KEYS: i64 = 50;
+pub const LARGE_RESULT_ROWS: usize = (FACT_FILES * FACT_ROWS * DIM_KEYS) as usize;
+
/// A small deterministic Parquet dataset: `facts(key, value)` joined to
/// `dims(key, name)`.
pub struct Fixture {
@@ -134,6 +136,11 @@ impl Fixture {
ORDER BY f.value"
}
+ pub fn large_result_query() -> &'static str {
+ "SELECT f.value, d.key FROM facts f CROSS JOIN dims d \
+ ORDER BY f.value, d.key"
+ }
+
/// The same query with a chaos UDF spliced into the WHERE clause.
///
/// `injection` is a complete `chaos_*(...)` call returning BOOLEAN. Its own
diff --git a/chaos-testing/tests/ha.rs b/chaos-testing/tests/ha.rs
index 4b5980aae..ce153fc1d 100644
--- a/chaos-testing/tests/ha.rs
+++ b/chaos-testing/tests/ha.rs
@@ -17,8 +17,8 @@
//! End-to-end high-availability scenarios against a real multi-process cluster.
//!
-//! Every scenario runs under both AQE settings. The AQE-on axis is where bugs are
-//! expected: a resubmitted stage under AQE is re-planned against runtime
+//! Query-recovery scenarios run under both AQE settings. The AQE-on axis is where
+//! bugs are expected: a resubmitted stage under AQE is re-planned against runtime
//! statistics, so a re-run map stage can come back with a different plan than the
//! one whose output was lost.
//!
@@ -28,8 +28,14 @@
mod common;
+use ballista::prelude::SessionConfigExt;
+use chaos_testing::fixture::{Fixture, LARGE_RESULT_ROWS};
use common::ChaosRun;
+use futures::TryStreamExt;
use rstest::rstest;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::time::timeout;
/// The cluster must agree with local DataFusion before any fault is injected.
/// Every recovery scenario asserts against this baseline, so if it is wrong,
@@ -57,8 +63,6 @@ fn ballista_chaos_query_baseline() -> &'static str {
chaos_testing::fixture::Fixture::baseline_query()
}
-use chaos_testing::fixture::Fixture;
-
/// Scenario A: one retryable (IO) fault, budget 1.
///
/// A single task attempt anywhere in the cluster faults; the budget is then
@@ -181,8 +185,6 @@ async fn panicking_task_fails_the_job_but_the_executor_survives(#[case] aqe: boo
assert_eq!(after, expected);
}
-use std::time::Duration;
-
/// Scenario D: SIGKILL an executor while it is running tasks.
///
/// `chaos_delay` holds the delayed scan stage open so the kill lands while
@@ -418,3 +420,54 @@ async fn killing_every_executor_terminates_the_job(#[case] aqe: bool) {
"failure should name the executor loss, got: {err}"
);
}
+
+/// Scenario H: SIGTERM an executor while the client is fetching final results.
+///
+/// Receiving the first batch means task execution has finished and the client is
+/// reading the final Flight response. Graceful shutdown must let that response
+/// finish before the executor exits.
+///
+/// Regression test for #2357.
+#[tokio::test]
+async fn final_results_survive_sigterm() {
+ let mut run = ChaosRun::start(false, 1).await;
+ let ctx = run.clone_ctx();
+ let config = ctx
+ .copied_config()
+ .with_target_partitions(1)
+ .with_ballista_override_create_grpc_client_endpoint(Arc::new(|endpoint| {
+ Ok(endpoint
+ .initial_stream_window_size(16 * 1024)
+ .http2_adaptive_window(false))
+ }));
+ *ctx.state_ref().write().config_mut() = config;
+
+ timeout(Duration::from_secs(120), async {
+ let df = ctx.sql(Fixture::large_result_query()).await.unwrap();
+ let mut stream = df.execute_stream().await.unwrap();
+ let mut rows = stream.try_next().await.unwrap().unwrap().num_rows();
+ assert!(
+ rows < LARGE_RESULT_ROWS,
+ "result must span multiple batches"
+ );
+
+ run.cluster.terminate_executor(0).unwrap();
+ run.cluster
+ .await_executor_count(0)
+ .await
+ .expect("scheduler must deregister the terminating executor");
+
+ while let Some(batch) = stream.try_next().await.unwrap_or_else(|e| {
+ panic!(
+ "result stream failed after {rows}/{LARGE_RESULT_ROWS} rows: {e}\n{}",
+ run.cluster.log_tails()
+ )
+ }) {
+ rows += batch.num_rows();
+ }
+ assert_eq!(rows, LARGE_RESULT_ROWS, "result was truncated");
+ assert!(run.cluster.await_executor_exit(0).await.unwrap().success());
+ })
+ .await
+ .expect("query and shutdown must not hang");
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org