You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2025/02/16 20:07:03 UTC
(datafusion-ballista) branch main updated: feat: notify scheduler when task fails (#1181)
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new faa05af9 feat: notify scheduler when task fails (#1181)
faa05af9 is described below
commit faa05af94408f4ec2a92f9af9581ca3bc4af3aea
Author: Marko Milenković <mi...@users.noreply.github.com>
AuthorDate: Sun Feb 16 20:06:58 2025 +0000
feat: notify scheduler when task fails (#1181)
---
ballista/executor/src/execution_loop.rs | 50 +++++++++++++++++++++++++++++++--
1 file changed, 47 insertions(+), 3 deletions(-)
diff --git a/ballista/executor/src/execution_loop.rs b/ballista/executor/src/execution_loop.rs
index 2094425d..df38a189 100644
--- a/ballista/executor/src/execution_loop.rs
+++ b/ballista/executor/src/execution_loop.rs
@@ -98,11 +98,16 @@ pub async fn poll_loop<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan>
let permit =
available_task_slots.clone().acquire_owned().await.unwrap();
+ let start_exec_time = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap()
+ .as_millis() as u64;
+
match run_received_task(
executor.clone(),
permit,
- task_status_sender,
- task,
+ task_status_sender.clone(),
+ task.clone(),
&codec,
&dedicated_executor,
)
@@ -110,7 +115,46 @@ pub async fn poll_loop<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan>
{
Ok(_) => {}
Err(e) => {
- warn!("Failed to run task: {:?}", e);
+ //
+ // notifying scheduler about task failure
+ // as scheduler expects notification.
+ //
+
+ let partition_id = PartitionId {
+ job_id: task.job_id.clone(),
+ stage_id: task.stage_id as usize,
+ partition_id: task.partition_id as usize,
+ };
+
+ warn!(
+ "Executor failed to run task: {:?}, error: {:?}",
+ partition_id, e
+ );
+
+ let end_exec_time = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap()
+ .as_millis()
+ as u64;
+
+ let task_execution_times = TaskExecutionTimes {
+ launch_time: task.launch_time,
+ start_exec_time,
+ end_exec_time,
+ };
+
+ // TODO: MM should we re-try message?
+ if let Err(error) = task_status_sender.send(as_task_status(
+ Err(e),
+ executor.metadata.id.clone(),
+ task.task_id as usize,
+ task.task_attempt_num as usize,
+ partition_id,
+ None,
+ task_execution_times,
+ )) {
+ warn!("failed to send task status: {:?}", error);
+ };
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org