You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2025/02/16 20:07:03 UTC

(datafusion-ballista) branch main updated: feat: notify scheduler when task fails (#1181)

This is an automated email from the ASF dual-hosted git repository.

agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git


The following commit(s) were added to refs/heads/main by this push:
     new faa05af9 feat: notify scheduler when task fails (#1181)
faa05af9 is described below

commit faa05af94408f4ec2a92f9af9581ca3bc4af3aea
Author: Marko Milenković <mi...@users.noreply.github.com>
AuthorDate: Sun Feb 16 20:06:58 2025 +0000

    feat: notify scheduler when task fails (#1181)
---
 ballista/executor/src/execution_loop.rs | 50 +++++++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 3 deletions(-)

diff --git a/ballista/executor/src/execution_loop.rs b/ballista/executor/src/execution_loop.rs
index 2094425d..df38a189 100644
--- a/ballista/executor/src/execution_loop.rs
+++ b/ballista/executor/src/execution_loop.rs
@@ -98,11 +98,16 @@ pub async fn poll_loop<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan>
                     let permit =
                         available_task_slots.clone().acquire_owned().await.unwrap();
 
+                    let start_exec_time = SystemTime::now()
+                        .duration_since(UNIX_EPOCH)
+                        .unwrap()
+                        .as_millis() as u64;
+
                     match run_received_task(
                         executor.clone(),
                         permit,
-                        task_status_sender,
-                        task,
+                        task_status_sender.clone(),
+                        task.clone(),
                         &codec,
                         &dedicated_executor,
                     )
@@ -110,7 +115,46 @@ pub async fn poll_loop<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan>
                     {
                         Ok(_) => {}
                         Err(e) => {
-                            warn!("Failed to run task: {:?}", e);
+                            //
+                            // notifying scheduler about task failure
+                            // as scheduler expects notification.
+                            //
+
+                            let partition_id = PartitionId {
+                                job_id: task.job_id.clone(),
+                                stage_id: task.stage_id as usize,
+                                partition_id: task.partition_id as usize,
+                            };
+
+                            warn!(
+                                "Executor failed to run task: {:?}, error: {:?}",
+                                partition_id, e
+                            );
+
+                            let end_exec_time = SystemTime::now()
+                                .duration_since(UNIX_EPOCH)
+                                .unwrap()
+                                .as_millis()
+                                as u64;
+
+                            let task_execution_times = TaskExecutionTimes {
+                                launch_time: task.launch_time,
+                                start_exec_time,
+                                end_exec_time,
+                            };
+
+                            // TODO: MM should we re-try message?
+                            if let Err(error) = task_status_sender.send(as_task_status(
+                                Err(e),
+                                executor.metadata.id.clone(),
+                                task.task_id as usize,
+                                task.task_attempt_num as usize,
+                                partition_id,
+                                None,
+                                task_execution_times,
+                            )) {
+                                warn!("failed to send task status: {:?}", error);
+                            };
                         }
                     }
                 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org