You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by mi...@apache.org on 2026/04/03 13:29:53 UTC
(datafusion-ballista) branch main updated: fix: remove unwrap from executor and improve task error handling (#1540)
This is an automated email from the ASF dual-hosted git repository.
milenkovicm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new 955aca5e5 fix: remove unwrap from executor and improve task error handling (#1540)
955aca5e5 is described below
commit 955aca5e5b1b1847aef7d02d2e8478a0b0590dd9
Author: Saj <ac...@gmail.com>
AuthorDate: Fri Apr 3 14:29:48 2026 +0100
fix: remove unwrap from executor and improve task error handling (#1540)
* Minor fix in the executor_server.rs file to improve error handling
* fix formatting
* Minor fix in the executor_server.rs file to improve error handling
* Restored comment removed
---------
Co-authored-by: Marko Milenković <mi...@users.noreply.github.com>
---
ballista/executor/src/executor_server.rs | 175 ++++++++++++++++++-------------
1 file changed, 101 insertions(+), 74 deletions(-)
diff --git a/ballista/executor/src/executor_server.rs b/ballista/executor/src/executor_server.rs
index c01f282ad..cff4666ee 100644
--- a/ballista/executor/src/executor_server.rs
+++ b/ballista/executor/src/executor_server.rs
@@ -345,7 +345,6 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
}
}
}
-
/// This method should not return Err. If task fails, a failure task status should be sent
/// to the channel to notify the scheduler.
async fn run_task(&self, task_identity: String, curator_task: CuratorTaskDefinition) {
@@ -369,83 +368,111 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorServer<T,
partition_id,
};
- let query_stage_exec = self
- .executor
- .execution_engine
- .create_query_stage_exec(
- job_id.clone(),
- stage_id,
- plan,
- &self.executor.work_dir,
- )
- .unwrap();
-
- let task_context = {
- let function_registry = task.function_registry;
- let runtime = self.executor.produce_runtime(&task.session_config).unwrap();
-
- Arc::new(TaskContext::new(
- Some(task_identity.clone()),
- task.session_id,
- task.session_config,
- function_registry.scalar_functions.clone(),
- function_registry.aggregate_functions.clone(),
- function_registry.window_functions.clone(),
- runtime,
- ))
- };
+ let exec = self.executor.execution_engine.create_query_stage_exec(
+ job_id.clone(),
+ stage_id,
+ plan,
+ &self.executor.work_dir,
+ );
- info!("Start to execute shuffle write for task {task_identity}");
+ let runtime = self.executor.produce_runtime(&task.session_config);
+
+ match (exec, runtime) {
+ (Ok(exec), Ok(runtime)) => {
+ let task_context = {
+ let function_registry = task.function_registry;
+
+ Arc::new(TaskContext::new(
+ Some(task_identity.clone()),
+ task.session_id,
+ task.session_config,
+ function_registry.scalar_functions.clone(),
+ function_registry.aggregate_functions.clone(),
+ function_registry.window_functions.clone(),
+ runtime,
+ ))
+ };
- let execution_result = self
- .executor
- .execute_query_stage(
- task_id,
- part.clone(),
- query_stage_exec.clone(),
- task_context,
- )
- .await;
- info!("Done with task {task_identity}");
- debug!("Statistics: {execution_result:?}");
-
- let plan_metrics = query_stage_exec.collect_plan_metrics();
- let operator_metrics = plan_metrics
- .into_iter()
- .map(|m| m.try_into())
- .collect::<Result<Vec<_>, BallistaError>>()
- .ok();
- let executor_id = &self.executor.metadata.id;
-
- let end_exec_time = SystemTime::now()
- .duration_since(UNIX_EPOCH)
- .unwrap()
- .as_millis() as u64;
- let task_execution_times = TaskExecutionTimes {
- launch_time: task.launch_time,
- start_exec_time,
- end_exec_time,
- };
+ info!("Execute task: {task_identity}");
- let task_status = as_task_status(
- execution_result,
- executor_id.clone(),
- task_id,
- stage_attempt_num,
- part,
- operator_metrics,
- task_execution_times,
- );
+ let execution_result = self
+ .executor
+ .execute_query_stage(
+ task_id,
+ part.clone(),
+ exec.clone(),
+ task_context,
+ )
+ .await;
+ info!("Done with task {task_identity}");
+ debug!("Task {task_identity} statistics: {execution_result:?}");
+
+ let plan_metrics = exec.collect_plan_metrics();
+ let operator_metrics = plan_metrics
+ .into_iter()
+ .map(|m| m.try_into())
+ .collect::<Result<Vec<_>, BallistaError>>()
+ .ok();
+ let executor_id = &self.executor.metadata.id;
+
+ let end_exec_time = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap()
+ .as_millis() as u64;
+ let task_execution_times = TaskExecutionTimes {
+ launch_time: task.launch_time,
+ start_exec_time,
+ end_exec_time,
+ };
- let scheduler_id = curator_task.scheduler_id;
- let task_status_sender = self.executor_env.tx_task_status.clone();
- task_status_sender
- .send(CuratorTaskStatus {
- scheduler_id,
- task_status,
- })
- .await
- .unwrap();
+ let task_status = as_task_status(
+ execution_result,
+ executor_id.clone(),
+ task_id,
+ stage_attempt_num,
+ part,
+ operator_metrics,
+ task_execution_times,
+ );
+
+ let _ = self
+ .executor_env
+ .tx_task_status
+ .send(CuratorTaskStatus {
+ scheduler_id: curator_task.scheduler_id,
+ task_status,
+ })
+ .await;
+ }
+ (Err(e), _) | (_, Err(e)) => {
+ let e = BallistaError::from(e);
+ let end_exec_time = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap()
+ .as_millis() as u64;
+ let task_status = as_task_status(
+ Err(e),
+ self.executor.metadata.id.clone(),
+ task_id,
+ stage_attempt_num,
+ part,
+ None,
+ TaskExecutionTimes {
+ launch_time: task.launch_time,
+ start_exec_time,
+ end_exec_time,
+ },
+ );
+ let _ = self
+ .executor_env
+ .tx_task_status
+ .send(CuratorTaskStatus {
+ scheduler_id: curator_task.scheduler_id,
+ task_status,
+ })
+ .await;
+ }
+ };
}
// TODO populate with real metrics
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org