You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by mi...@apache.org on 2026/04/03 13:29:53 UTC

(datafusion-ballista) branch main updated: fix: remove unwrap from executor and improve task error handling (#1540)

This is an automated email from the ASF dual-hosted git repository.

milenkovicm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git


The following commit(s) were added to refs/heads/main by this push:
     new 955aca5e5 fix: remove unwrap from executor and improve task error handling (#1540)
955aca5e5 is described below

commit 955aca5e5b1b1847aef7d02d2e8478a0b0590dd9
Author: Saj <ac...@gmail.com>
AuthorDate: Fri Apr 3 14:29:48 2026 +0100

    fix: remove unwrap from executor and improve task error handling (#1540)
    
    * Minor fix  in the executor_server.rs file to improve error handling
    
    * fix formatting
    
    * Minor fix  in the executor_server.rs file to improve error handling
    
    * Restored comment removed
    
    ---------
    
    Co-authored-by: Marko Milenković <mi...@users.noreply.github.com>
---
 ballista/executor/src/executor_server.rs | 175 ++++++++++++++++++-------------
 1 file changed, 101 insertions(+), 74 deletions(-)

diff --git a/ballista/executor/src/executor_server.rs b/ballista/executor/src/executor_server.rs
index c01f282ad..cff4666ee 100644
--- a/ballista/executor/src/executor_server.rs
+++ b/ballista/executor/src/executor_server.rs
@@ -345,7 +345,6 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
             }
         }
     }
-
     /// This method should not return Err. If task fails, a failure task status should be sent
     /// to the channel to notify the scheduler.
     async fn run_task(&self, task_identity: String, curator_task: CuratorTaskDefinition) {
@@ -369,83 +368,111 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
             partition_id,
         };
 
-        let query_stage_exec = self
-            .executor
-            .execution_engine
-            .create_query_stage_exec(
-                job_id.clone(),
-                stage_id,
-                plan,
-                &self.executor.work_dir,
-            )
-            .unwrap();
-
-        let task_context = {
-            let function_registry = task.function_registry;
-            let runtime = self.executor.produce_runtime(&task.session_config).unwrap();
-
-            Arc::new(TaskContext::new(
-                Some(task_identity.clone()),
-                task.session_id,
-                task.session_config,
-                function_registry.scalar_functions.clone(),
-                function_registry.aggregate_functions.clone(),
-                function_registry.window_functions.clone(),
-                runtime,
-            ))
-        };
+        let exec = self.executor.execution_engine.create_query_stage_exec(
+            job_id.clone(),
+            stage_id,
+            plan,
+            &self.executor.work_dir,
+        );
 
-        info!("Start to execute shuffle write for task {task_identity}");
+        let runtime = self.executor.produce_runtime(&task.session_config);
+
+        match (exec, runtime) {
+            (Ok(exec), Ok(runtime)) => {
+                let task_context = {
+                    let function_registry = task.function_registry;
+
+                    Arc::new(TaskContext::new(
+                        Some(task_identity.clone()),
+                        task.session_id,
+                        task.session_config,
+                        function_registry.scalar_functions.clone(),
+                        function_registry.aggregate_functions.clone(),
+                        function_registry.window_functions.clone(),
+                        runtime,
+                    ))
+                };
 
-        let execution_result = self
-            .executor
-            .execute_query_stage(
-                task_id,
-                part.clone(),
-                query_stage_exec.clone(),
-                task_context,
-            )
-            .await;
-        info!("Done with task {task_identity}");
-        debug!("Statistics: {execution_result:?}");
-
-        let plan_metrics = query_stage_exec.collect_plan_metrics();
-        let operator_metrics = plan_metrics
-            .into_iter()
-            .map(|m| m.try_into())
-            .collect::<Result<Vec<_>, BallistaError>>()
-            .ok();
-        let executor_id = &self.executor.metadata.id;
-
-        let end_exec_time = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .unwrap()
-            .as_millis() as u64;
-        let task_execution_times = TaskExecutionTimes {
-            launch_time: task.launch_time,
-            start_exec_time,
-            end_exec_time,
-        };
+                info!("Execute task: {task_identity}");
 
-        let task_status = as_task_status(
-            execution_result,
-            executor_id.clone(),
-            task_id,
-            stage_attempt_num,
-            part,
-            operator_metrics,
-            task_execution_times,
-        );
+                let execution_result = self
+                    .executor
+                    .execute_query_stage(
+                        task_id,
+                        part.clone(),
+                        exec.clone(),
+                        task_context,
+                    )
+                    .await;
+                info!("Done with task {task_identity}");
+                debug!("Task {task_identity} statistics: {execution_result:?}");
+
+                let plan_metrics = exec.collect_plan_metrics();
+                let operator_metrics = plan_metrics
+                    .into_iter()
+                    .map(|m| m.try_into())
+                    .collect::<Result<Vec<_>, BallistaError>>()
+                    .ok();
+                let executor_id = &self.executor.metadata.id;
+
+                let end_exec_time = SystemTime::now()
+                    .duration_since(UNIX_EPOCH)
+                    .unwrap()
+                    .as_millis() as u64;
+                let task_execution_times = TaskExecutionTimes {
+                    launch_time: task.launch_time,
+                    start_exec_time,
+                    end_exec_time,
+                };
 
-        let scheduler_id = curator_task.scheduler_id;
-        let task_status_sender = self.executor_env.tx_task_status.clone();
-        task_status_sender
-            .send(CuratorTaskStatus {
-                scheduler_id,
-                task_status,
-            })
-            .await
-            .unwrap();
+                let task_status = as_task_status(
+                    execution_result,
+                    executor_id.clone(),
+                    task_id,
+                    stage_attempt_num,
+                    part,
+                    operator_metrics,
+                    task_execution_times,
+                );
+
+                let _ = self
+                    .executor_env
+                    .tx_task_status
+                    .send(CuratorTaskStatus {
+                        scheduler_id: curator_task.scheduler_id,
+                        task_status,
+                    })
+                    .await;
+            }
+            (Err(e), _) | (_, Err(e)) => {
+                let e = BallistaError::from(e);
+                let end_exec_time = SystemTime::now()
+                    .duration_since(UNIX_EPOCH)
+                    .unwrap()
+                    .as_millis() as u64;
+                let task_status = as_task_status(
+                    Err(e),
+                    self.executor.metadata.id.clone(),
+                    task_id,
+                    stage_attempt_num,
+                    part,
+                    None,
+                    TaskExecutionTimes {
+                        launch_time: task.launch_time,
+                        start_exec_time,
+                        end_exec_time,
+                    },
+                );
+                let _ = self
+                    .executor_env
+                    .tx_task_status
+                    .send(CuratorTaskStatus {
+                        scheduler_id: curator_task.scheduler_id,
+                        task_status,
+                    })
+                    .await;
+            }
+        };
     }
 
     // TODO populate with real metrics


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org