You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by mi...@apache.org on 2026/10/15 15:37:02 UTC

(datafusion-ballista) branch main updated: fix(executor): drain Flight before shutdown cleanup (#2448)

This is an automated email from the ASF dual-hosted git repository.

milenkovicm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git


The following commit(s) were added to refs/heads/main by this push:
     new d15184476 fix(executor): drain Flight before shutdown cleanup (#2448)
d15184476 is described below

commit d151844766ea034902b14bc20ff312fc3a77871b
Author: Dan <da...@gmail.com>
AuthorDate: Thu Oct 15 16:36:57 2026 +0100

    fix(executor): drain Flight before shutdown cleanup (#2448)
---
 Cargo.lock                                |   1 +
 ballista/executor/src/config.rs           |   4 +
 ballista/executor/src/executor_process.rs | 144 +++++++++++++++++++-----------
 chaos-testing/Cargo.toml                  |   1 +
 chaos-testing/src/bin/chaos-executor.rs   |  10 +++
 chaos-testing/src/cluster.rs              |  41 ++++++++-
 chaos-testing/src/fixture.rs              |   7 ++
 chaos-testing/tests/ha.rs                 |  65 ++++++++++++--
 8 files changed, 212 insertions(+), 61 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 562607cb6..902aef808 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1069,6 +1069,7 @@ dependencies = [
  "ballista-scheduler",
  "datafusion",
  "env_logger",
+ "futures",
  "log",
  "nix 0.31.3",
  "reqwest 0.13.5",
diff --git a/ballista/executor/src/config.rs b/ballista/executor/src/config.rs
index a812f487c..d8965a5e3 100644
--- a/ballista/executor/src/config.rs
+++ b/ballista/executor/src/config.rs
@@ -104,6 +104,9 @@ pub struct Config {
         help = "How long to try connecting to scheduler before failing. Set to zero to fail after first attempt."
     )]
     pub scheduler_connect_timeout_seconds: u16,
+    /// Maximum time in seconds for executor graceful shutdown (0 = wait indefinitely).
+    #[arg(long, default_value_t = 10)]
+    pub graceful_shutdown_timeout_seconds: u64,
     /// Directory for storing temporary shuffle data and IPC files.
     #[arg(long, help = "Directory for temporary IPC files")]
     pub work_dir: Option<String>,
@@ -264,6 +267,7 @@ impl TryFrom<Config> for ExecutorProcessConfig {
             scheduler_host: opt.scheduler_host,
             scheduler_port: opt.scheduler_port,
             scheduler_connect_timeout_seconds: opt.scheduler_connect_timeout_seconds,
+            graceful_shutdown_timeout_seconds: opt.graceful_shutdown_timeout_seconds,
             vcores,
             task_scheduling_policy: opt.task_scheduling_policy,
             work_dir: opt.work_dir,
diff --git a/ballista/executor/src/executor_process.rs b/ballista/executor/src/executor_process.rs
index 1f72c925f..1e64220e0 100644
--- a/ballista/executor/src/executor_process.rs
+++ b/ballista/executor/src/executor_process.rs
@@ -298,6 +298,9 @@ pub struct ExecutorProcessConfig {
     pub scheduler_port: u16,
     /// Timeout in seconds for establishing scheduler connection.
     pub scheduler_connect_timeout_seconds: u16,
+    /// Maximum time in seconds for scheduler notification, task and Flight draining,
+    /// and shuffle cleanup after shutdown is requested. `0` waits indefinitely.
+    pub graceful_shutdown_timeout_seconds: u64,
     /// Virtual cores advertised by this executor to the scheduler.
     pub vcores: usize,
     /// Task scheduling policy (pull-staged or push-staged).
@@ -403,6 +406,7 @@ impl Default for ExecutorProcessConfig {
             scheduler_host: "localhost".into(),
             scheduler_port: 50050,
             scheduler_connect_timeout_seconds: 0,
+            graceful_shutdown_timeout_seconds: 10,
             vcores: std::thread::available_parallelism().unwrap().get(),
             task_scheduling_policy: Default::default(),
             log_dir: None,
@@ -727,10 +731,12 @@ async fn start_executor_process_inner(
     // Graceful shutdown notification
     let shutdown_notification = ShutdownNotifier::new();
     let flight_work_dir = work_dir.clone();
+    let clean_job_data_on_shutdown = opt.job_data_clean_up_interval_seconds > 0;
 
-    if opt.job_data_clean_up_interval_seconds > 0 {
+    if clean_job_data_on_shutdown {
         let mut interval_time =
             time::interval(Duration::from_secs(opt.job_data_clean_up_interval_seconds));
+        let cleaner_work_dir = work_dir.clone();
 
         let mut shuffle_cleaner_shutdown = shutdown_notification.subscribe_for_shutdown();
         let shuffle_cleaner_complete = shutdown_notification.shutdown_complete_tx.clone();
@@ -740,18 +746,12 @@ async fn start_executor_process_inner(
             while !shuffle_cleaner_shutdown.is_shutdown() {
                 tokio::select! {
                     _ = interval_time.tick() => {
-                            if let Err(e) = clean_shuffle_data_loop(&work_dir, job_data_ttl_seconds).await
+                            if let Err(e) = clean_shuffle_data_loop(&cleaner_work_dir, job_data_ttl_seconds).await
                         {
                             error!("Ballista executor fail to clean_shuffle_data {e:?}")
                         }
                         },
                     _ = shuffle_cleaner_shutdown.recv() => {
-                        if let Err(e) = clean_all_shuffle_data(&work_dir).await
-                        {
-                            error!("Ballista executor fail to clean_shuffle_data {e:?}")
-                        } else {
-                            info!("Shuffle data cleaned.");
-                        }
                         drop(shuffle_cleaner_complete);
                         return;
                     }
@@ -793,14 +793,21 @@ async fn start_executor_process_inner(
             );
         }
         _ => {
-            service_handlers.push(tokio::spawn(execution_loop::poll_loop(
+            let poll_loop = execution_loop::poll_loop(
                 scheduler.clone(),
                 executor.clone(),
                 default_codec,
                 None, // poll_now_notify
                 None, // free_vcores: use internal semaphore
                 health,
-            )));
+            );
+            let mut poll_shutdown = shutdown_notification.subscribe_for_shutdown();
+            service_handlers.push(tokio::spawn(async move {
+                tokio::select! {
+                    result = poll_loop => result,
+                    _ = poll_shutdown.recv() => Ok(()),
+                }
+            }));
         }
     };
     let shutdown = shutdown_notification.subscribe_for_shutdown();
@@ -860,57 +867,86 @@ async fn start_executor_process_inner(
     info!("setting executor to TERMINATING status");
     TERMINATING.store(true, Ordering::Release);
 
-    if notify_scheduler {
-        // Send a heartbeat to update status of executor to `Fenced`. This should signal to the
-        // scheduler to no longer schedule tasks on this executor
-        if let Err(error) = scheduler
-            .heart_beat_from_executor(HeartBeatParams {
-                executor_id: executor_id.clone(),
-                metrics: vec![],
-                status: Some(ExecutorStatus {
-                    status: Some(Status::Terminating(String::default())),
-                }),
-                metadata: Some(executor_meta),
-            })
-            .await
-        {
-            error!("error sending heartbeat with fenced status: {error:?}");
-        }
-
-        // TODO we probably don't need a separate rpc call for this....
-        if let Err(error) = scheduler
-            .executor_stopped(ExecutorStoppedParams {
-                executor_id,
-                reason: stop_reason,
-            })
-            .await
-        {
-            error!("ExecutorStopped grpc failed: {error:?}");
-        }
-
-        // Wait for tasks to drain
-        tasks_drained.await;
-    }
-
-    // Extract the `shutdown_complete` receiver and transmitter
-    // explicitly drop `shutdown_transmitter`. This is important, as the
-    // `.await` below would otherwise never complete.
     let ShutdownNotifier {
         mut shutdown_complete_rx,
         shutdown_complete_tx,
         notify_shutdown,
-        ..
     } = shutdown_notification;
 
-    // When `notify_shutdown` is dropped, all components which have `subscribe`d will
-    // receive the shutdown signal and can exit
-    let _ = notify_shutdown.send(());
-    drop(notify_shutdown);
-    // Drop final `Sender` so the `Receiver` below can complete
-    drop(shutdown_complete_tx);
+    let graceful_shutdown = async {
+        if notify_scheduler {
+            // Send a heartbeat to update status of executor to `Fenced`. This should signal to the
+            // scheduler to no longer schedule tasks on this executor
+            if let Err(error) = scheduler
+                .heart_beat_from_executor(HeartBeatParams {
+                    executor_id: executor_id.clone(),
+                    metrics: vec![],
+                    status: Some(ExecutorStatus {
+                        status: Some(Status::Terminating(String::default())),
+                    }),
+                    metadata: Some(executor_meta),
+                })
+                .await
+            {
+                error!("error sending heartbeat with fenced status: {error:?}");
+            }
+
+            // TODO we probably don't need a separate rpc call for this....
+            if let Err(error) = scheduler
+                .executor_stopped(ExecutorStoppedParams {
+                    executor_id,
+                    reason: stop_reason,
+                })
+                .await
+            {
+                error!("ExecutorStopped grpc failed: {error:?}");
+            }
+
+            // Wait for tasks to drain
+            tasks_drained.await;
+        }
 
-    // Wait for all related components to finish the shutdown processing.
-    let _ = shutdown_complete_rx.recv().await;
+        // Signal all subscribed components to shut down.
+        let _ = notify_shutdown.send(());
+        // Drop final `Sender` so the `Receiver` below can complete
+        drop(shutdown_complete_tx);
+
+        // Wait for all related components to finish the shutdown processing.
+        let _ = shutdown_complete_rx.recv().await;
+        // Drain the servers, including Flight, before removing their shuffle data.
+        while !service_handlers.is_empty() {
+            if let Err(error) = check_services(&mut service_handlers).await {
+                error!("Executor service failed during shutdown: {error:?}");
+            }
+        }
+        if clean_job_data_on_shutdown {
+            if let Err(e) = clean_all_shuffle_data(&work_dir).await {
+                error!("Ballista executor fail to clean_shuffle_data {e:?}")
+            } else {
+                info!("Shuffle data cleaned.");
+            }
+        }
+    };
+
+    let shutdown_timeout = Duration::from_secs(opt.graceful_shutdown_timeout_seconds);
+    if shutdown_timeout.is_zero() {
+        graceful_shutdown.await;
+    } else if time::timeout(shutdown_timeout, graceful_shutdown)
+        .await
+        .is_err()
+    {
+        let message = format!(
+            "Executor graceful shutdown timed out after {} seconds",
+            shutdown_timeout.as_secs()
+        );
+        warn!("{message}; aborting remaining services");
+        // A scheduler RPC or task drain may have timed out before broadcasting shutdown.
+        let _ = notify_shutdown.send(());
+        for service in &service_handlers {
+            service.abort();
+        }
+        return Err(BallistaError::General(message));
+    }
     info!("Executor stopped.");
     Ok(())
 }
diff --git a/chaos-testing/Cargo.toml b/chaos-testing/Cargo.toml
index 477adbc70..2100d64bf 100644
--- a/chaos-testing/Cargo.toml
+++ b/chaos-testing/Cargo.toml
@@ -47,6 +47,7 @@ tempfile = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread", "process", "sync", "time", "net", "signal"] }
 
 [dev-dependencies]
+futures = { workspace = true }
 rstest = { workspace = true }
 
 [lib]
diff --git a/chaos-testing/src/bin/chaos-executor.rs b/chaos-testing/src/bin/chaos-executor.rs
index f8d3a765d..48346fcbe 100644
--- a/chaos-testing/src/bin/chaos-executor.rs
+++ b/chaos-testing/src/bin/chaos-executor.rs
@@ -52,6 +52,16 @@ async fn main() -> ballista_core::error::Result<()> {
             .unwrap_or_else(|_| "127.0.0.1".into()),
         scheduler_port: env_u16("CHAOS_SCHEDULER_PORT"),
         scheduler_connect_timeout_seconds: 10,
+        graceful_shutdown_timeout_seconds: std::env::var(
+            "CHAOS_GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS",
+        )
+        .ok()
+        .map(|value| {
+            value
+                .parse()
+                .expect("Graceful shutdown timeout must be a u64")
+        })
+        .unwrap_or(10),
         vcores: std::env::var("CHAOS_CONCURRENT_TASKS")
             .ok()
             .and_then(|v| v.parse().ok())
diff --git a/chaos-testing/src/cluster.rs b/chaos-testing/src/cluster.rs
index 5f03252cc..1623d80fa 100644
--- a/chaos-testing/src/cluster.rs
+++ b/chaos-testing/src/cluster.rs
@@ -20,7 +20,7 @@ use nix::fcntl::{Flock, FlockArg};
 use std::fs::{File, OpenOptions};
 use std::net::TcpListener;
 use std::path::{Path, PathBuf};
-use std::process::{Child, Command, Stdio};
+use std::process::{Child, Command, ExitStatus, Stdio};
 use std::sync::{Arc, OnceLock};
 use std::time::{Duration, Instant};
 use tokio::sync::{Mutex, OwnedMutexGuard};
@@ -99,6 +99,7 @@ pub struct TestClusterBuilder {
     stage_max_failures: usize,
     concurrent_tasks: usize,
     no_executors_grace_seconds: u64,
+    graceful_shutdown_timeout_seconds: u64,
     result_fetch: Option<ResultFetch>,
 }
 
@@ -116,6 +117,8 @@ impl Default for TestClusterBuilder {
             // Ballista's default is 30s. A short grace makes the total-loss
             // scenario fail the job a second or so after the reap.
             no_executors_grace_seconds: 1,
+            // Let test timeouts bound shutdown so slow CI can finish draining.
+            graceful_shutdown_timeout_seconds: 0,
             result_fetch: None,
         }
     }
@@ -427,6 +430,10 @@ impl TestCluster {
             )
             .env("CHAOS_WORK_DIR", work_dir.display().to_string())
             .env("CHAOS_HEARTBEAT_SECONDS", "1")
+            .env(
+                "CHAOS_GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS",
+                self.builder.graceful_shutdown_timeout_seconds.to_string(),
+            )
             .env(
                 "RUST_LOG",
                 std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into()),
@@ -725,6 +732,38 @@ impl TestCluster {
         Ok(())
     }
 
+    /// Request graceful shutdown without waiting, so a client can keep streaming.
+    pub fn terminate_executor(&self, index: usize) -> Result<(), String> {
+        use nix::sys::signal::{Signal, kill};
+        use nix::unistd::Pid;
+
+        let pid = self.executors[index].child.id();
+        kill(Pid::from_raw(pid as i32), Signal::SIGTERM).map_err(|e| e.to_string())
+    }
+
+    pub async fn await_executor_exit(
+        &mut self,
+        index: usize,
+    ) -> Result<ExitStatus, String> {
+        let deadline = Instant::now() + Duration::from_secs(30);
+        loop {
+            if let Some(status) = self.executors[index]
+                .child
+                .try_wait()
+                .map_err(|e| e.to_string())?
+            {
+                return Ok(status);
+            }
+            if Instant::now() > deadline {
+                return Err(format!(
+                    "executor {index} did not exit\n{}",
+                    self.log_tails()
+                ));
+            }
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+    }
+
     pub async fn await_successful_shuffle_output(
         &self,
         job_id: &str,
diff --git a/chaos-testing/src/fixture.rs b/chaos-testing/src/fixture.rs
index f8c6d8784..4fcbd2b61 100644
--- a/chaos-testing/src/fixture.rs
+++ b/chaos-testing/src/fixture.rs
@@ -32,6 +32,8 @@ const FACT_FILES: i64 = 8;
 /// is what creates the multi-stage plan the HA paths need.
 const DIM_KEYS: i64 = 50;
 
+pub const LARGE_RESULT_ROWS: usize = (FACT_FILES * FACT_ROWS * DIM_KEYS) as usize;
+
 /// A small deterministic Parquet dataset: `facts(key, value)` joined to
 /// `dims(key, name)`.
 pub struct Fixture {
@@ -134,6 +136,11 @@ impl Fixture {
          ORDER BY f.value"
     }
 
+    pub fn large_result_query() -> &'static str {
+        "SELECT f.value, d.key FROM facts f CROSS JOIN dims d \
+         ORDER BY f.value, d.key"
+    }
+
     /// The same query with a chaos UDF spliced into the WHERE clause.
     ///
     /// `injection` is a complete `chaos_*(...)` call returning BOOLEAN. Its own
diff --git a/chaos-testing/tests/ha.rs b/chaos-testing/tests/ha.rs
index 4b5980aae..ce153fc1d 100644
--- a/chaos-testing/tests/ha.rs
+++ b/chaos-testing/tests/ha.rs
@@ -17,8 +17,8 @@
 
 //! End-to-end high-availability scenarios against a real multi-process cluster.
 //!
-//! Every scenario runs under both AQE settings. The AQE-on axis is where bugs are
-//! expected: a resubmitted stage under AQE is re-planned against runtime
+//! Query-recovery scenarios run under both AQE settings. The AQE-on axis is where
+//! bugs are expected: a resubmitted stage under AQE is re-planned against runtime
 //! statistics, so a re-run map stage can come back with a different plan than the
 //! one whose output was lost.
 //!
@@ -28,8 +28,14 @@
 
 mod common;
 
+use ballista::prelude::SessionConfigExt;
+use chaos_testing::fixture::{Fixture, LARGE_RESULT_ROWS};
 use common::ChaosRun;
+use futures::TryStreamExt;
 use rstest::rstest;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::time::timeout;
 
 /// The cluster must agree with local DataFusion before any fault is injected.
 /// Every recovery scenario asserts against this baseline, so if it is wrong,
@@ -57,8 +63,6 @@ fn ballista_chaos_query_baseline() -> &'static str {
     chaos_testing::fixture::Fixture::baseline_query()
 }
 
-use chaos_testing::fixture::Fixture;
-
 /// Scenario A: one retryable (IO) fault, budget 1.
 ///
 /// A single task attempt anywhere in the cluster faults; the budget is then
@@ -181,8 +185,6 @@ async fn panicking_task_fails_the_job_but_the_executor_survives(#[case] aqe: boo
     assert_eq!(after, expected);
 }
 
-use std::time::Duration;
-
 /// Scenario D: SIGKILL an executor while it is running tasks.
 ///
 /// `chaos_delay` holds the delayed scan stage open so the kill lands while
@@ -418,3 +420,54 @@ async fn killing_every_executor_terminates_the_job(#[case] aqe: bool) {
         "failure should name the executor loss, got: {err}"
     );
 }
+
+/// Scenario H: SIGTERM an executor while the client is fetching final results.
+///
+/// Receiving the first batch means task execution has finished and the client is
+/// reading the final Flight response. Graceful shutdown must let that response
+/// finish before the executor exits.
+///
+/// Regression test for #2357.
+#[tokio::test]
+async fn final_results_survive_sigterm() {
+    let mut run = ChaosRun::start(false, 1).await;
+    let ctx = run.clone_ctx();
+    let config = ctx
+        .copied_config()
+        .with_target_partitions(1)
+        .with_ballista_override_create_grpc_client_endpoint(Arc::new(|endpoint| {
+            Ok(endpoint
+                .initial_stream_window_size(16 * 1024)
+                .http2_adaptive_window(false))
+        }));
+    *ctx.state_ref().write().config_mut() = config;
+
+    timeout(Duration::from_secs(120), async {
+        let df = ctx.sql(Fixture::large_result_query()).await.unwrap();
+        let mut stream = df.execute_stream().await.unwrap();
+        let mut rows = stream.try_next().await.unwrap().unwrap().num_rows();
+        assert!(
+            rows < LARGE_RESULT_ROWS,
+            "result must span multiple batches"
+        );
+
+        run.cluster.terminate_executor(0).unwrap();
+        run.cluster
+            .await_executor_count(0)
+            .await
+            .expect("scheduler must deregister the terminating executor");
+
+        while let Some(batch) = stream.try_next().await.unwrap_or_else(|e| {
+            panic!(
+                "result stream failed after {rows}/{LARGE_RESULT_ROWS} rows: {e}\n{}",
+                run.cluster.log_tails()
+            )
+        }) {
+            rows += batch.num_rows();
+        }
+        assert_eq!(rows, LARGE_RESULT_ROWS, "result was truncated");
+        assert!(run.cluster.await_executor_exit(0).await.unwrap().success());
+    })
+    .await
+    .expect("query and shutdown must not hang");
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org