You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ag...@apache.org on 2022/08/25 17:52:32 UTC
[arrow-ballista] branch master updated: Add job cancellation (#146)
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-ballista.git
The following commit(s) were added to refs/heads/master by this push:
new 7b4ad2f5 Add job cancellation (#146)
7b4ad2f5 is described below
commit 7b4ad2f5a7fa9cfe01d07c451f80f73fef2601bf
Author: Dan Harris <13...@users.noreply.github.com>
AuthorDate: Thu Aug 25 13:52:27 2022 -0400
Add job cancellation (#146)
* Add job cancellation
* Use util method to create grpc connection
---
ballista/rust/core/proto/ballista.proto | 20 ++
ballista/rust/core/src/error.rs | 9 +
ballista/rust/executor/src/executor.rs | 224 ++++++++++++++++++++-
ballista/rust/executor/src/executor_server.rs | 33 ++-
ballista/rust/scheduler/Cargo.toml | 1 +
.../rust/scheduler/src/scheduler_server/grpc.rs | 29 ++-
.../rust/scheduler/src/scheduler_server/mod.rs | 15 +-
.../rust/scheduler/src/state/execution_graph.rs | 41 ++++
.../rust/scheduler/src/state/executor_manager.rs | 35 ++++
ballista/rust/scheduler/src/state/task_manager.rs | 83 +++++++-
10 files changed, 473 insertions(+), 17 deletions(-)
diff --git a/ballista/rust/core/proto/ballista.proto b/ballista/rust/core/proto/ballista.proto
index 0ebc922d..f0c0a3aa 100644
--- a/ballista/rust/core/proto/ballista.proto
+++ b/ballista/rust/core/proto/ballista.proto
@@ -763,6 +763,14 @@ message FilePartitionMetadata {
repeated string filename = 1;
}
+message CancelJobParams {
+ string job_id = 1;
+}
+
+message CancelJobResult {
+ bool cancelled = 1;
+}
+
message LaunchTaskParams {
// Allow to launch a task set to an executor at once
repeated TaskDefinition task = 1;
@@ -773,6 +781,14 @@ message LaunchTaskResult {
// TODO when part of the task set are scheduled successfully
}
+message CancelTasksParams {
+ repeated PartitionId partition_id = 1;
+}
+
+message CancelTasksResult {
+ bool cancelled = 1;
+}
+
service SchedulerGrpc {
// Executors must poll the scheduler for heartbeat and to receive tasks
rpc PollWork (PollWorkParams) returns (PollWorkResult) {}
@@ -790,10 +806,14 @@ service SchedulerGrpc {
rpc ExecuteQuery (ExecuteQueryParams) returns (ExecuteQueryResult) {}
rpc GetJobStatus (GetJobStatusParams) returns (GetJobStatusResult) {}
+
+ rpc CancelJob (CancelJobParams) returns (CancelJobResult) {}
}
service ExecutorGrpc {
rpc LaunchTask (LaunchTaskParams) returns (LaunchTaskResult) {}
rpc StopExecutor (StopExecutorParams) returns (StopExecutorResult) {}
+
+ rpc CancelTasks (CancelTasksParams) returns (CancelTasksResult) {}
}
diff --git a/ballista/rust/core/src/error.rs b/ballista/rust/core/src/error.rs
index ba6cdc74..dd4f9dda 100644
--- a/ballista/rust/core/src/error.rs
+++ b/ballista/rust/core/src/error.rs
@@ -25,6 +25,7 @@ use std::{
use datafusion::arrow::error::ArrowError;
use datafusion::error::DataFusionError;
+use futures::future::Aborted;
use sqlparser::parser;
pub type Result<T> = result::Result<T, BallistaError>;
@@ -47,6 +48,7 @@ pub enum BallistaError {
TonicError(tonic::transport::Error),
GrpcError(tonic::Status),
TokioError(tokio::task::JoinError),
+ Cancelled,
}
#[allow(clippy::from_over_into)]
@@ -150,6 +152,12 @@ impl From<datafusion_proto::to_proto::Error> for BallistaError {
}
}
+impl From<futures::future::Aborted> for BallistaError {
+ fn from(_: Aborted) -> Self {
+ BallistaError::Cancelled
+ }
+}
+
impl Display for BallistaError {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
match self {
@@ -178,6 +186,7 @@ impl Display for BallistaError {
write!(f, "Internal Ballista error: {}", desc)
}
BallistaError::TokioError(desc) => write!(f, "Tokio join error: {}", desc),
+ BallistaError::Cancelled => write!(f, "Task cancelled"),
}
}
}
diff --git a/ballista/rust/executor/src/executor.rs b/ballista/rust/executor/src/executor.rs
index 34af3427..ff1cde80 100644
--- a/ballista/rust/executor/src/executor.rs
+++ b/ballista/rust/executor/src/executor.rs
@@ -32,8 +32,15 @@ use datafusion::execution::runtime_env::RuntimeEnv;
use datafusion::physical_plan::udaf::AggregateUDF;
use datafusion::physical_plan::udf::ScalarUDF;
use datafusion::physical_plan::{ExecutionPlan, Partitioning};
+use futures::future::AbortHandle;
+
+use ballista_core::serde::scheduler::PartitionId;
+use tokio::sync::Mutex;
+
+type AbortHandles = Arc<Mutex<HashMap<PartitionId, AbortHandle>>>;
/// Ballista executor
+#[derive(Clone)]
pub struct Executor {
/// Metadata
pub metadata: ExecutorRegistration,
@@ -55,6 +62,9 @@ pub struct Executor {
/// Concurrent tasks can run in executor
pub concurrent_tasks: usize,
+
+ /// Handles to abort executing tasks
+ abort_handles: AbortHandles,
}
impl Executor {
@@ -75,6 +85,7 @@ impl Executor {
runtime,
metrics_collector,
concurrent_tasks,
+ abort_handles: Default::default(),
}
}
}
@@ -92,7 +103,30 @@ impl Executor {
task_ctx: Arc<TaskContext>,
_shuffle_output_partitioning: Option<Partitioning>,
) -> Result<Vec<protobuf::ShuffleWritePartition>, BallistaError> {
- let partitions = shuffle_writer.execute_shuffle_write(part, task_ctx).await?;
+ let (task, abort_handle) = futures::future::abortable(
+ shuffle_writer.execute_shuffle_write(part, task_ctx),
+ );
+
+ {
+ let mut abort_handles = self.abort_handles.lock().await;
+ abort_handles.insert(
+ PartitionId {
+ job_id: job_id.clone(),
+ stage_id,
+ partition_id: part,
+ },
+ abort_handle,
+ );
+ }
+
+ let partitions = task.await??;
+
+ self.abort_handles.lock().await.remove(&PartitionId {
+ job_id: job_id.clone(),
+ stage_id,
+ partition_id: part,
+ });
+
self.metrics_collector
.record_stage(&job_id, stage_id, part, shuffle_writer);
@@ -126,7 +160,195 @@ impl Executor {
Ok(Arc::new(exec))
}
+ pub async fn cancel_task(
+ &self,
+ job_id: String,
+ stage_id: usize,
+ partition_id: usize,
+ ) -> Result<bool, BallistaError> {
+ if let Some(handle) = self.abort_handles.lock().await.remove(&PartitionId {
+ job_id,
+ stage_id,
+ partition_id,
+ }) {
+ handle.abort();
+ Ok(true)
+ } else {
+ Ok(false)
+ }
+ }
+
pub fn work_dir(&self) -> &str {
&self.work_dir
}
}
+
+#[cfg(test)]
+mod test {
+ use crate::executor::Executor;
+ use crate::metrics::LoggingMetricsCollector;
+ use arrow::datatypes::{Schema, SchemaRef};
+ use arrow::error::ArrowError;
+ use arrow::record_batch::RecordBatch;
+ use ballista_core::execution_plans::ShuffleWriterExec;
+ use ballista_core::serde::protobuf::ExecutorRegistration;
+ use datafusion::execution::context::TaskContext;
+
+ use datafusion::physical_expr::PhysicalSortExpr;
+ use datafusion::physical_plan::{
+ ExecutionPlan, Partitioning, RecordBatchStream, SendableRecordBatchStream,
+ Statistics,
+ };
+ use datafusion::prelude::SessionContext;
+ use futures::Stream;
+ use std::any::Any;
+ use std::pin::Pin;
+ use std::sync::Arc;
+ use std::task::{Context, Poll};
+ use std::time::Duration;
+ use tempfile::TempDir;
+
+ /// A RecordBatchStream that will never terminate
+ struct NeverendingRecordBatchStream;
+
+ impl RecordBatchStream for NeverendingRecordBatchStream {
+ fn schema(&self) -> SchemaRef {
+ Arc::new(Schema::empty())
+ }
+ }
+
+ impl Stream for NeverendingRecordBatchStream {
+ type Item = Result<RecordBatch, ArrowError>;
+
+ fn poll_next(
+ self: Pin<&mut Self>,
+ _cx: &mut Context<'_>,
+ ) -> Poll<Option<Self::Item>> {
+ Poll::Pending
+ }
+ }
+
+ /// An ExecutionPlan which will never terminate
+ #[derive(Debug)]
+ pub struct NeverendingOperator;
+
+ impl ExecutionPlan for NeverendingOperator {
+ fn as_any(&self) -> &dyn Any {
+ self
+ }
+
+ fn schema(&self) -> SchemaRef {
+ Arc::new(Schema::empty())
+ }
+
+ fn output_partitioning(&self) -> Partitioning {
+ Partitioning::UnknownPartitioning(1)
+ }
+
+ fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> {
+ None
+ }
+
+ fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+ vec![]
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ _children: Vec<Arc<dyn ExecutionPlan>>,
+ ) -> datafusion::common::Result<Arc<dyn ExecutionPlan>> {
+ Ok(self)
+ }
+
+ fn execute(
+ &self,
+ _partition: usize,
+ _context: Arc<TaskContext>,
+ ) -> datafusion::common::Result<SendableRecordBatchStream> {
+ Ok(Box::pin(NeverendingRecordBatchStream))
+ }
+
+ fn statistics(&self) -> Statistics {
+ Statistics::default()
+ }
+ }
+
+ #[tokio::test]
+ async fn test_task_cancellation() {
+ let work_dir = TempDir::new()
+ .unwrap()
+ .into_path()
+ .into_os_string()
+ .into_string()
+ .unwrap();
+
+ let shuffle_write = ShuffleWriterExec::try_new(
+ "job-id".to_owned(),
+ 1,
+ Arc::new(NeverendingOperator),
+ work_dir.clone(),
+ None,
+ )
+ .expect("creating shuffle writer");
+
+ let executor_registration = ExecutorRegistration {
+ id: "executor".to_string(),
+ port: 0,
+ grpc_port: 0,
+ specification: None,
+ optional_host: None,
+ };
+
+ let ctx = SessionContext::new();
+
+ let executor = Executor::new(
+ executor_registration,
+ &work_dir,
+ ctx.runtime_env(),
+ Arc::new(LoggingMetricsCollector {}),
+ 2,
+ );
+
+ let (sender, receiver) = tokio::sync::oneshot::channel();
+
+ // Spawn our non-terminating task on a separate fiber.
+ let executor_clone = executor.clone();
+ tokio::task::spawn(async move {
+ let task_result = executor_clone
+ .execute_shuffle_write(
+ "job-id".to_owned(),
+ 1,
+ 0,
+ Arc::new(shuffle_write),
+ ctx.task_ctx(),
+ None,
+ )
+ .await;
+ sender.send(task_result).expect("sending result");
+ });
+
+ // Now cancel the task. We can only cancel once the task has been executed and has an `AbortHandle` registered, so
+ // poll until that happens.
+ for _ in 0..20 {
+ if executor
+ .cancel_task("job-id".to_owned(), 1, 0)
+ .await
+ .expect("cancelling task")
+ {
+ break;
+ } else {
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ }
+
+ // Wait for our task to complete
+ let result = tokio::time::timeout(Duration::from_secs(5), receiver).await;
+
+ // Make sure the task didn't timeout
+ assert!(result.is_ok());
+
+ // Make sure the actual task failed
+ let inner_result = result.unwrap().unwrap();
+ assert!(inner_result.is_err());
+ }
+}
diff --git a/ballista/rust/executor/src/executor_server.rs b/ballista/rust/executor/src/executor_server.rs
index 8f0e758f..dc2724fd 100644
--- a/ballista/rust/executor/src/executor_server.rs
+++ b/ballista/rust/executor/src/executor_server.rs
@@ -35,9 +35,9 @@ use ballista_core::serde::protobuf::executor_grpc_server::{
use ballista_core::serde::protobuf::executor_registration::OptionalHost;
use ballista_core::serde::protobuf::scheduler_grpc_client::SchedulerGrpcClient;
use ballista_core::serde::protobuf::{
- HeartBeatParams, LaunchTaskParams, LaunchTaskResult, RegisterExecutorParams,
- StopExecutorParams, StopExecutorResult, TaskDefinition, TaskStatus,
- UpdateTaskStatusParams,
+ CancelTasksParams, CancelTasksResult, HeartBeatParams, LaunchTaskParams,
+ LaunchTaskResult, RegisterExecutorParams, StopExecutorParams, StopExecutorResult,
+ TaskDefinition, TaskStatus, UpdateTaskStatusParams,
};
use ballista_core::serde::scheduler::ExecutorState;
use ballista_core::serde::{AsExecutionPlan, BallistaCodec};
@@ -505,4 +505,31 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorGrpc
stop_sender.send(force).await.unwrap();
Ok(Response::new(StopExecutorResult {}))
}
+
+ async fn cancel_tasks(
+ &self,
+ request: Request<CancelTasksParams>,
+ ) -> Result<Response<CancelTasksResult>, Status> {
+ let partitions = request.into_inner().partition_id;
+ info!("Cancelling partition tasks for {:?}", partitions);
+
+ let mut cancelled = true;
+
+ for partition in partitions {
+ if let Err(e) = self
+ .executor
+ .cancel_task(
+ partition.job_id,
+ partition.stage_id as usize,
+ partition.partition_id as usize,
+ )
+ .await
+ {
+ error!("Error cancelling task: {:?}", e);
+ cancelled = false;
+ }
+ }
+
+ Ok(Response::new(CancelTasksResult { cancelled }))
+ }
}
diff --git a/ballista/rust/scheduler/Cargo.toml b/ballista/rust/scheduler/Cargo.toml
index 826daa9d..9f098038 100644
--- a/ballista/rust/scheduler/Cargo.toml
+++ b/ballista/rust/scheduler/Cargo.toml
@@ -50,6 +50,7 @@ futures = "0.3"
http = "0.2"
http-body = "0.4"
hyper = "0.14.4"
+itertools = "0.10.3"
log = "0.4"
object_store = "0.3.0"
parking_lot = "0.12"
diff --git a/ballista/rust/scheduler/src/scheduler_server/grpc.rs b/ballista/rust/scheduler/src/scheduler_server/grpc.rs
index 1ec73c04..af788dc4 100644
--- a/ballista/rust/scheduler/src/scheduler_server/grpc.rs
+++ b/ballista/rust/scheduler/src/scheduler_server/grpc.rs
@@ -22,10 +22,11 @@ use ballista_core::serde::protobuf::execute_query_params::{OptionalSessionId, Qu
use ballista_core::serde::protobuf::executor_registration::OptionalHost;
use ballista_core::serde::protobuf::scheduler_grpc_server::SchedulerGrpc;
use ballista_core::serde::protobuf::{
- ExecuteQueryParams, ExecuteQueryResult, ExecutorHeartbeat, GetFileMetadataParams,
- GetFileMetadataResult, GetJobStatusParams, GetJobStatusResult, HeartBeatParams,
- HeartBeatResult, PollWorkParams, PollWorkResult, RegisterExecutorParams,
- RegisterExecutorResult, UpdateTaskStatusParams, UpdateTaskStatusResult,
+ CancelJobParams, CancelJobResult, ExecuteQueryParams, ExecuteQueryResult,
+ ExecutorHeartbeat, GetFileMetadataParams, GetFileMetadataResult, GetJobStatusParams,
+ GetJobStatusResult, HeartBeatParams, HeartBeatResult, PollWorkParams, PollWorkResult,
+ RegisterExecutorParams, RegisterExecutorResult, UpdateTaskStatusParams,
+ UpdateTaskStatusResult,
};
use ballista_core::serde::scheduler::{ExecutorData, ExecutorMetadata};
use ballista_core::serde::AsExecutionPlan;
@@ -489,6 +490,26 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerGrpc
}
}
}
+
+ async fn cancel_job(
+ &self,
+ request: Request<CancelJobParams>,
+ ) -> Result<Response<CancelJobResult>, Status> {
+ let job_id = request.into_inner().job_id;
+ info!("Received cancellation request for job {}", job_id);
+
+ self.state
+ .task_manager
+ .cancel_job(&job_id, &self.state.executor_manager)
+ .await
+ .map_err(|e| {
+ let msg = format!("Error cancelling job {}: {:?}", job_id, e);
+ error!("{}", msg);
+ Status::internal(msg)
+ })?;
+
+ Ok(Response::new(CancelJobResult { cancelled: true }))
+ }
}
#[cfg(all(test, feature = "sled"))]
diff --git a/ballista/rust/scheduler/src/scheduler_server/mod.rs b/ballista/rust/scheduler/src/scheduler_server/mod.rs
index e84f595a..a672c597 100644
--- a/ballista/rust/scheduler/src/scheduler_server/mod.rs
+++ b/ballista/rust/scheduler/src/scheduler_server/mod.rs
@@ -34,6 +34,7 @@ use crate::scheduler_server::event::{QueryStageSchedulerEvent, SchedulerServerEv
use crate::scheduler_server::event_loop::SchedulerServerEventAction;
use crate::scheduler_server::query_stage_scheduler::QueryStageScheduler;
use crate::state::backend::StateBackendClient;
+use crate::state::executor_manager::ExecutorReservation;
use crate::state::SchedulerState;
// include the generated protobuf source as a submodule
@@ -225,7 +226,19 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerServer<T
"Failed to update {} task statuses for executor {}: {:?}",
num_status, executor_id, e
);
- // TODO what do we do here?
+ // In case task update fails, make sure to free reservations
+ if let Some(event_loop) = self.event_loop.as_ref() {
+ let mut reservations = vec![];
+ for _ in 0..num_status {
+ reservations
+ .push(ExecutorReservation::new_free(executor_id.to_owned()));
+ }
+
+ event_loop
+ .get_sender()?
+ .post_event(SchedulerServerEvent::Offer(reservations))
+ .await?;
+ }
}
}
diff --git a/ballista/rust/scheduler/src/state/execution_graph.rs b/ballista/rust/scheduler/src/state/execution_graph.rs
index e3599092..b831ffe9 100644
--- a/ballista/rust/scheduler/src/state/execution_graph.rs
+++ b/ballista/rust/scheduler/src/state/execution_graph.rs
@@ -317,6 +317,29 @@ impl ExecutionGraph {
.sum()
}
+ /// Return all currently running tasks along with the executor ID on which they are assigned
+ pub fn running_tasks(&self) -> Vec<(PartitionId, String)> {
+ self.stages
+ .iter()
+ .flat_map(|(_, stage)| {
+ stage
+ .running_tasks()
+ .iter()
+ .map(|(stage_id, partition, executor_id)| {
+ (
+ PartitionId {
+ job_id: self.job_id.clone(),
+ stage_id: *stage_id,
+ partition_id: *partition,
+ },
+ executor_id.clone(),
+ )
+ })
+ .collect::<Vec<(PartitionId, String)>>()
+ })
+ .collect::<Vec<(PartitionId, String)>>()
+ }
+
/// Get next task that can be assigned to the given executor.
/// This method should only be called when the resulting task is immediately
/// being launched as the status will be set to Running and it will not be
@@ -748,6 +771,24 @@ impl ExecutionStage {
}
}
+ /// Returns a vector of currently running tasks in this stage
+ pub fn running_tasks(&self) -> Vec<(usize, usize, String)> {
+ if self.resolved {
+ self.task_statuses
+ .iter()
+ .enumerate()
+ .filter_map(|(partition, status)| match status {
+ Some(task_status::Status::Running(RunningTask { executor_id })) => {
+ Some((self.stage_id, partition, executor_id.clone()))
+ }
+ _ => None,
+ })
+ .collect()
+ } else {
+ vec![]
+ }
+ }
+
/// Returns true if all inputs are complete and we can resolve all
/// UnresolvedShuffleExec operators to ShuffleReadExec
pub fn resolvable(&self) -> bool {
diff --git a/ballista/rust/scheduler/src/state/executor_manager.rs b/ballista/rust/scheduler/src/state/executor_manager.rs
index ad2a4389..4c5449c8 100644
--- a/ballista/rust/scheduler/src/state/executor_manager.rs
+++ b/ballista/rust/scheduler/src/state/executor_manager.rs
@@ -23,12 +23,17 @@ use crate::state::{decode_into, decode_protobuf, encode_protobuf, with_lock};
use ballista_core::error::{BallistaError, Result};
use ballista_core::serde::protobuf;
+use ballista_core::serde::protobuf::executor_grpc_client::ExecutorGrpcClient;
use ballista_core::serde::scheduler::{ExecutorData, ExecutorMetadata};
+use ballista_core::utils::create_grpc_client_connection;
use futures::StreamExt;
use log::{debug, info};
use parking_lot::RwLock;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
+use tonic::transport::Channel;
+
+type ExecutorClients = Arc<RwLock<HashMap<String, ExecutorGrpcClient<Channel>>>>;
/// Represents a task slot that is reserved (i.e. available for scheduling but not visible to the
/// rest of the system).
@@ -71,6 +76,7 @@ pub(crate) struct ExecutorManager {
state: Arc<dyn StateBackendClient>,
executor_metadata: Arc<RwLock<HashMap<String, ExecutorMetadata>>>,
executors_heartbeat: Arc<RwLock<HashMap<String, protobuf::ExecutorHeartbeat>>>,
+ clients: ExecutorClients,
}
impl ExecutorManager {
@@ -79,6 +85,7 @@ impl ExecutorManager {
state,
executor_metadata: Arc::new(RwLock::new(HashMap::new())),
executors_heartbeat: Arc::new(RwLock::new(HashMap::new())),
+ clients: Default::default(),
}
}
@@ -194,6 +201,34 @@ impl ExecutorManager {
.await
}
+ pub async fn get_client(
+ &self,
+ executor_id: &str,
+ ) -> Result<ExecutorGrpcClient<Channel>> {
+ let client = {
+ let clients = self.clients.read();
+ clients.get(executor_id).cloned()
+ };
+
+ if let Some(client) = client {
+ Ok(client)
+ } else {
+ let executor_metadata = self.get_executor_metadata(executor_id).await?;
+ let executor_url = format!(
+ "http://{}:{}",
+ executor_metadata.host, executor_metadata.grpc_port
+ );
+ let connection = create_grpc_client_connection(executor_url).await?;
+ let client = ExecutorGrpcClient::new(connection);
+
+ {
+ let mut clients = self.clients.write();
+ clients.insert(executor_id.to_owned(), client.clone());
+ }
+ Ok(client)
+ }
+ }
+
/// Get a list of all executors along with the timestamp of their last recorded heartbeat
pub async fn get_executor_state(&self) -> Result<Vec<(ExecutorMetadata, Duration)>> {
let heartbeat_timestamps: Vec<(String, u64)> = {
diff --git a/ballista/rust/scheduler/src/state/task_manager.rs b/ballista/rust/scheduler/src/state/task_manager.rs
index bb0db021..58c49a3b 100644
--- a/ballista/rust/scheduler/src/state/task_manager.rs
+++ b/ballista/rust/scheduler/src/state/task_manager.rs
@@ -17,9 +17,9 @@
use crate::scheduler_server::event::QueryStageSchedulerEvent;
use crate::scheduler_server::SessionBuilder;
-use crate::state::backend::{Keyspace, StateBackendClient};
+use crate::state::backend::{Keyspace, Lock, StateBackendClient};
use crate::state::execution_graph::{ExecutionGraph, Task};
-use crate::state::executor_manager::ExecutorReservation;
+use crate::state::executor_manager::{ExecutorManager, ExecutorReservation};
use crate::state::{decode_protobuf, encode_protobuf, with_lock};
use ballista_core::config::BallistaConfig;
#[cfg(not(test))]
@@ -29,7 +29,7 @@ use ballista_core::serde::protobuf::executor_grpc_client::ExecutorGrpcClient;
use crate::state::session_manager::create_datafusion_context;
use ballista_core::serde::protobuf::{
- self, job_status, FailedJob, JobStatus, PartitionId, TaskDefinition, TaskStatus,
+ self, job_status, CancelTasksParams, FailedJob, JobStatus, TaskDefinition, TaskStatus,
};
use ballista_core::serde::scheduler::to_proto::hash_partitioning_to_proto;
use ballista_core::serde::scheduler::ExecutorMetadata;
@@ -37,7 +37,7 @@ use ballista_core::serde::{AsExecutionPlan, BallistaCodec};
use datafusion::physical_plan::ExecutionPlan;
use datafusion::prelude::SessionContext;
use datafusion_proto::logical_plan::AsLogicalPlan;
-use log::{debug, info, warn};
+use log::{debug, error, info, warn};
use rand::distributions::Alphanumeric;
use rand::{thread_rng, Rng};
use std::collections::{HashMap, HashSet};
@@ -79,6 +79,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
plan: Arc<dyn ExecutionPlan>,
) -> Result<()> {
let graph = ExecutionGraph::new(job_id, session_id, plan)?;
+ info!("Submitting execution graph: {:?}", graph);
self.state
.put(
Keyspace::ActiveJobs,
@@ -283,7 +284,6 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
reservation.executor_id, task
);
assignments.push((executor_id.clone(), task));
- // locks.push(lock);
graphs.insert(job_id, graph);
assigned = true;
break;
@@ -333,11 +333,78 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
.await
}
+ pub(crate) async fn cancel_job(
+ &self,
+ job_id: &str,
+ executor_manager: &ExecutorManager,
+ ) -> Result<()> {
+ let lock = self.state.lock(Keyspace::ActiveJobs, "").await?;
+
+ let running_tasks = self
+ .get_execution_graph(job_id)
+ .await
+ .map(|graph| graph.running_tasks())
+ .unwrap_or_else(|_| vec![]);
+
+ info!(
+ "Cancelling {} running tasks for job {}",
+ running_tasks.len(),
+ job_id
+ );
+
+ self.fail_job_inner(lock, job_id, "Cancelled".to_owned())
+ .await?;
+
+ let mut tasks: HashMap<&str, Vec<protobuf::PartitionId>> = Default::default();
+
+ for (partition, executor_id) in &running_tasks {
+ if let Some(parts) = tasks.get_mut(executor_id.as_str()) {
+ parts.push(protobuf::PartitionId {
+ job_id: job_id.to_owned(),
+ stage_id: partition.stage_id as u32,
+ partition_id: partition.partition_id as u32,
+ })
+ } else {
+ tasks.insert(
+ executor_id.as_str(),
+ vec![protobuf::PartitionId {
+ job_id: job_id.to_owned(),
+ stage_id: partition.stage_id as u32,
+ partition_id: partition.partition_id as u32,
+ }],
+ );
+ }
+ }
+
+ for (executor_id, partitions) in tasks {
+ if let Ok(mut client) = executor_manager.get_client(executor_id).await {
+ client
+ .cancel_tasks(CancelTasksParams {
+ partition_id: partitions,
+ })
+ .await?;
+ } else {
+ error!("Failed to get client for executor ID {}", executor_id)
+ }
+ }
+
+ Ok(())
+ }
+
/// Mark a job as failed. This will create a key under the FailedJobs keyspace
/// and remove the job from ActiveJobs or QueuedJobs
/// TODO this should be atomic
pub async fn fail_job(&self, job_id: &str, error_message: String) -> Result<()> {
let lock = self.state.lock(Keyspace::ActiveJobs, "").await?;
+ self.fail_job_inner(lock, job_id, error_message).await
+ }
+
+ async fn fail_job_inner(
+ &self,
+ lock: Box<dyn Lock>,
+ job_id: &str,
+ error_message: String,
+ ) -> Result<()> {
with_lock(lock, self.state.delete(Keyspace::ActiveJobs, job_id)).await?;
let status = JobStatus {
@@ -354,7 +421,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
#[cfg(not(test))]
/// Launch the given task on the specified executor
- pub async fn launch_task(
+ pub(crate) async fn launch_task(
&self,
executor: &ExecutorMetadata,
task: Task,
@@ -398,7 +465,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
/// In unit tests, we do not have actual executors running, so it simplifies things to just noop.
#[cfg(test)]
- pub async fn launch_task(
+ pub(crate) async fn launch_task(
&self,
_executor: &ExecutorMetadata,
_task: Task,
@@ -426,7 +493,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
hash_partitioning_to_proto(task.output_partitioning.as_ref())?;
let task_definition = TaskDefinition {
- task_id: Some(PartitionId {
+ task_id: Some(protobuf::PartitionId {
job_id: task.partition.job_id.clone(),
stage_id: task.partition.stage_id as u32,
partition_id: task.partition.partition_id as u32,