You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ag...@apache.org on 2022/08/25 17:52:32 UTC

[arrow-ballista] branch master updated: Add job cancellation (#146)

This is an automated email from the ASF dual-hosted git repository.

agrove pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-ballista.git


The following commit(s) were added to refs/heads/master by this push:
     new 7b4ad2f5 Add job cancellation (#146)
7b4ad2f5 is described below

commit 7b4ad2f5a7fa9cfe01d07c451f80f73fef2601bf
Author: Dan Harris <13...@users.noreply.github.com>
AuthorDate: Thu Aug 25 13:52:27 2022 -0400

    Add job cancellation (#146)
    
    * Add job cancellation
    
    * Use util method to create grpc connection
---
 ballista/rust/core/proto/ballista.proto            |  20 ++
 ballista/rust/core/src/error.rs                    |   9 +
 ballista/rust/executor/src/executor.rs             | 224 ++++++++++++++++++++-
 ballista/rust/executor/src/executor_server.rs      |  33 ++-
 ballista/rust/scheduler/Cargo.toml                 |   1 +
 .../rust/scheduler/src/scheduler_server/grpc.rs    |  29 ++-
 .../rust/scheduler/src/scheduler_server/mod.rs     |  15 +-
 .../rust/scheduler/src/state/execution_graph.rs    |  41 ++++
 .../rust/scheduler/src/state/executor_manager.rs   |  35 ++++
 ballista/rust/scheduler/src/state/task_manager.rs  |  83 +++++++-
 10 files changed, 473 insertions(+), 17 deletions(-)

diff --git a/ballista/rust/core/proto/ballista.proto b/ballista/rust/core/proto/ballista.proto
index 0ebc922d..f0c0a3aa 100644
--- a/ballista/rust/core/proto/ballista.proto
+++ b/ballista/rust/core/proto/ballista.proto
@@ -763,6 +763,14 @@ message FilePartitionMetadata {
   repeated string filename = 1;
 }
 
+message CancelJobParams {
+  string job_id = 1;
+}
+
+message CancelJobResult {
+  bool cancelled = 1;
+}
+
 message LaunchTaskParams {
   // Allow to launch a task set to an executor at once
   repeated TaskDefinition task = 1;
@@ -773,6 +781,14 @@ message LaunchTaskResult {
   // TODO when part of the task set are scheduled successfully
 }
 
+message CancelTasksParams {
+  repeated PartitionId partition_id = 1;
+}
+
+message CancelTasksResult {
+  bool cancelled = 1;
+}
+
 service SchedulerGrpc {
   // Executors must poll the scheduler for heartbeat and to receive tasks
   rpc PollWork (PollWorkParams) returns (PollWorkResult) {}
@@ -790,10 +806,14 @@ service SchedulerGrpc {
   rpc ExecuteQuery (ExecuteQueryParams) returns (ExecuteQueryResult) {}
 
   rpc GetJobStatus (GetJobStatusParams) returns (GetJobStatusResult) {}
+
+  rpc CancelJob (CancelJobParams) returns (CancelJobResult) {}
 }
 
 service ExecutorGrpc {
   rpc LaunchTask (LaunchTaskParams) returns (LaunchTaskResult) {}
 
   rpc StopExecutor (StopExecutorParams) returns (StopExecutorResult) {}
+
+  rpc CancelTasks (CancelTasksParams) returns (CancelTasksResult) {}
 }
diff --git a/ballista/rust/core/src/error.rs b/ballista/rust/core/src/error.rs
index ba6cdc74..dd4f9dda 100644
--- a/ballista/rust/core/src/error.rs
+++ b/ballista/rust/core/src/error.rs
@@ -25,6 +25,7 @@ use std::{
 
 use datafusion::arrow::error::ArrowError;
 use datafusion::error::DataFusionError;
+use futures::future::Aborted;
 use sqlparser::parser;
 
 pub type Result<T> = result::Result<T, BallistaError>;
@@ -47,6 +48,7 @@ pub enum BallistaError {
     TonicError(tonic::transport::Error),
     GrpcError(tonic::Status),
     TokioError(tokio::task::JoinError),
+    Cancelled,
 }
 
 #[allow(clippy::from_over_into)]
@@ -150,6 +152,12 @@ impl From<datafusion_proto::to_proto::Error> for BallistaError {
     }
 }
 
+impl From<futures::future::Aborted> for BallistaError {
+    fn from(_: Aborted) -> Self {
+        BallistaError::Cancelled
+    }
+}
+
 impl Display for BallistaError {
     fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
         match self {
@@ -178,6 +186,7 @@ impl Display for BallistaError {
                 write!(f, "Internal Ballista error: {}", desc)
             }
             BallistaError::TokioError(desc) => write!(f, "Tokio join error: {}", desc),
+            BallistaError::Cancelled => write!(f, "Task cancelled"),
         }
     }
 }
diff --git a/ballista/rust/executor/src/executor.rs b/ballista/rust/executor/src/executor.rs
index 34af3427..ff1cde80 100644
--- a/ballista/rust/executor/src/executor.rs
+++ b/ballista/rust/executor/src/executor.rs
@@ -32,8 +32,15 @@ use datafusion::execution::runtime_env::RuntimeEnv;
 use datafusion::physical_plan::udaf::AggregateUDF;
 use datafusion::physical_plan::udf::ScalarUDF;
 use datafusion::physical_plan::{ExecutionPlan, Partitioning};
+use futures::future::AbortHandle;
+
+use ballista_core::serde::scheduler::PartitionId;
+use tokio::sync::Mutex;
+
+type AbortHandles = Arc<Mutex<HashMap<PartitionId, AbortHandle>>>;
 
 /// Ballista executor
+#[derive(Clone)]
 pub struct Executor {
     /// Metadata
     pub metadata: ExecutorRegistration,
@@ -55,6 +62,9 @@ pub struct Executor {
 
     /// Concurrent tasks can run in executor
     pub concurrent_tasks: usize,
+
+    /// Handles to abort executing tasks
+    abort_handles: AbortHandles,
 }
 
 impl Executor {
@@ -75,6 +85,7 @@ impl Executor {
             runtime,
             metrics_collector,
             concurrent_tasks,
+            abort_handles: Default::default(),
         }
     }
 }
@@ -92,7 +103,30 @@ impl Executor {
         task_ctx: Arc<TaskContext>,
         _shuffle_output_partitioning: Option<Partitioning>,
     ) -> Result<Vec<protobuf::ShuffleWritePartition>, BallistaError> {
-        let partitions = shuffle_writer.execute_shuffle_write(part, task_ctx).await?;
+        let (task, abort_handle) = futures::future::abortable(
+            shuffle_writer.execute_shuffle_write(part, task_ctx),
+        );
+
+        {
+            let mut abort_handles = self.abort_handles.lock().await;
+            abort_handles.insert(
+                PartitionId {
+                    job_id: job_id.clone(),
+                    stage_id,
+                    partition_id: part,
+                },
+                abort_handle,
+            );
+        }
+
+        let partitions = task.await??;
+
+        self.abort_handles.lock().await.remove(&PartitionId {
+            job_id: job_id.clone(),
+            stage_id,
+            partition_id: part,
+        });
+
         self.metrics_collector
             .record_stage(&job_id, stage_id, part, shuffle_writer);
 
@@ -126,7 +160,195 @@ impl Executor {
         Ok(Arc::new(exec))
     }
 
+    pub async fn cancel_task(
+        &self,
+        job_id: String,
+        stage_id: usize,
+        partition_id: usize,
+    ) -> Result<bool, BallistaError> {
+        if let Some(handle) = self.abort_handles.lock().await.remove(&PartitionId {
+            job_id,
+            stage_id,
+            partition_id,
+        }) {
+            handle.abort();
+            Ok(true)
+        } else {
+            Ok(false)
+        }
+    }
+
     pub fn work_dir(&self) -> &str {
         &self.work_dir
     }
 }
+
+#[cfg(test)]
+mod test {
+    use crate::executor::Executor;
+    use crate::metrics::LoggingMetricsCollector;
+    use arrow::datatypes::{Schema, SchemaRef};
+    use arrow::error::ArrowError;
+    use arrow::record_batch::RecordBatch;
+    use ballista_core::execution_plans::ShuffleWriterExec;
+    use ballista_core::serde::protobuf::ExecutorRegistration;
+    use datafusion::execution::context::TaskContext;
+
+    use datafusion::physical_expr::PhysicalSortExpr;
+    use datafusion::physical_plan::{
+        ExecutionPlan, Partitioning, RecordBatchStream, SendableRecordBatchStream,
+        Statistics,
+    };
+    use datafusion::prelude::SessionContext;
+    use futures::Stream;
+    use std::any::Any;
+    use std::pin::Pin;
+    use std::sync::Arc;
+    use std::task::{Context, Poll};
+    use std::time::Duration;
+    use tempfile::TempDir;
+
+    /// A RecordBatchStream that will never terminate
+    struct NeverendingRecordBatchStream;
+
+    impl RecordBatchStream for NeverendingRecordBatchStream {
+        fn schema(&self) -> SchemaRef {
+            Arc::new(Schema::empty())
+        }
+    }
+
+    impl Stream for NeverendingRecordBatchStream {
+        type Item = Result<RecordBatch, ArrowError>;
+
+        fn poll_next(
+            self: Pin<&mut Self>,
+            _cx: &mut Context<'_>,
+        ) -> Poll<Option<Self::Item>> {
+            Poll::Pending
+        }
+    }
+
+    /// An ExecutionPlan which will never terminate
+    #[derive(Debug)]
+    pub struct NeverendingOperator;
+
+    impl ExecutionPlan for NeverendingOperator {
+        fn as_any(&self) -> &dyn Any {
+            self
+        }
+
+        fn schema(&self) -> SchemaRef {
+            Arc::new(Schema::empty())
+        }
+
+        fn output_partitioning(&self) -> Partitioning {
+            Partitioning::UnknownPartitioning(1)
+        }
+
+        fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> {
+            None
+        }
+
+        fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+            vec![]
+        }
+
+        fn with_new_children(
+            self: Arc<Self>,
+            _children: Vec<Arc<dyn ExecutionPlan>>,
+        ) -> datafusion::common::Result<Arc<dyn ExecutionPlan>> {
+            Ok(self)
+        }
+
+        fn execute(
+            &self,
+            _partition: usize,
+            _context: Arc<TaskContext>,
+        ) -> datafusion::common::Result<SendableRecordBatchStream> {
+            Ok(Box::pin(NeverendingRecordBatchStream))
+        }
+
+        fn statistics(&self) -> Statistics {
+            Statistics::default()
+        }
+    }
+
+    #[tokio::test]
+    async fn test_task_cancellation() {
+        let work_dir = TempDir::new()
+            .unwrap()
+            .into_path()
+            .into_os_string()
+            .into_string()
+            .unwrap();
+
+        let shuffle_write = ShuffleWriterExec::try_new(
+            "job-id".to_owned(),
+            1,
+            Arc::new(NeverendingOperator),
+            work_dir.clone(),
+            None,
+        )
+        .expect("creating shuffle writer");
+
+        let executor_registration = ExecutorRegistration {
+            id: "executor".to_string(),
+            port: 0,
+            grpc_port: 0,
+            specification: None,
+            optional_host: None,
+        };
+
+        let ctx = SessionContext::new();
+
+        let executor = Executor::new(
+            executor_registration,
+            &work_dir,
+            ctx.runtime_env(),
+            Arc::new(LoggingMetricsCollector {}),
+            2,
+        );
+
+        let (sender, receiver) = tokio::sync::oneshot::channel();
+
+        // Spawn our non-terminating task on a separate fiber.
+        let executor_clone = executor.clone();
+        tokio::task::spawn(async move {
+            let task_result = executor_clone
+                .execute_shuffle_write(
+                    "job-id".to_owned(),
+                    1,
+                    0,
+                    Arc::new(shuffle_write),
+                    ctx.task_ctx(),
+                    None,
+                )
+                .await;
+            sender.send(task_result).expect("sending result");
+        });
+
+        // Now cancel the task. We can only cancel once the task has been executed and has an `AbortHandle` registered, so
+        // poll until that happens.
+        for _ in 0..20 {
+            if executor
+                .cancel_task("job-id".to_owned(), 1, 0)
+                .await
+                .expect("cancelling task")
+            {
+                break;
+            } else {
+                tokio::time::sleep(Duration::from_millis(50)).await;
+            }
+        }
+
+        // Wait for our task to complete
+        let result = tokio::time::timeout(Duration::from_secs(5), receiver).await;
+
+        // Make sure the task didn't timeout
+        assert!(result.is_ok());
+
+        // Make sure the actual task failed
+        let inner_result = result.unwrap().unwrap();
+        assert!(inner_result.is_err());
+    }
+}
diff --git a/ballista/rust/executor/src/executor_server.rs b/ballista/rust/executor/src/executor_server.rs
index 8f0e758f..dc2724fd 100644
--- a/ballista/rust/executor/src/executor_server.rs
+++ b/ballista/rust/executor/src/executor_server.rs
@@ -35,9 +35,9 @@ use ballista_core::serde::protobuf::executor_grpc_server::{
 use ballista_core::serde::protobuf::executor_registration::OptionalHost;
 use ballista_core::serde::protobuf::scheduler_grpc_client::SchedulerGrpcClient;
 use ballista_core::serde::protobuf::{
-    HeartBeatParams, LaunchTaskParams, LaunchTaskResult, RegisterExecutorParams,
-    StopExecutorParams, StopExecutorResult, TaskDefinition, TaskStatus,
-    UpdateTaskStatusParams,
+    CancelTasksParams, CancelTasksResult, HeartBeatParams, LaunchTaskParams,
+    LaunchTaskResult, RegisterExecutorParams, StopExecutorParams, StopExecutorResult,
+    TaskDefinition, TaskStatus, UpdateTaskStatusParams,
 };
 use ballista_core::serde::scheduler::ExecutorState;
 use ballista_core::serde::{AsExecutionPlan, BallistaCodec};
@@ -505,4 +505,31 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> ExecutorGrpc
         stop_sender.send(force).await.unwrap();
         Ok(Response::new(StopExecutorResult {}))
     }
+
+    async fn cancel_tasks(
+        &self,
+        request: Request<CancelTasksParams>,
+    ) -> Result<Response<CancelTasksResult>, Status> {
+        let partitions = request.into_inner().partition_id;
+        info!("Cancelling partition tasks for {:?}", partitions);
+
+        let mut cancelled = true;
+
+        for partition in partitions {
+            if let Err(e) = self
+                .executor
+                .cancel_task(
+                    partition.job_id,
+                    partition.stage_id as usize,
+                    partition.partition_id as usize,
+                )
+                .await
+            {
+                error!("Error cancelling task: {:?}", e);
+                cancelled = false;
+            }
+        }
+
+        Ok(Response::new(CancelTasksResult { cancelled }))
+    }
 }
diff --git a/ballista/rust/scheduler/Cargo.toml b/ballista/rust/scheduler/Cargo.toml
index 826daa9d..9f098038 100644
--- a/ballista/rust/scheduler/Cargo.toml
+++ b/ballista/rust/scheduler/Cargo.toml
@@ -50,6 +50,7 @@ futures = "0.3"
 http = "0.2"
 http-body = "0.4"
 hyper = "0.14.4"
+itertools = "0.10.3"
 log = "0.4"
 object_store = "0.3.0"
 parking_lot = "0.12"
diff --git a/ballista/rust/scheduler/src/scheduler_server/grpc.rs b/ballista/rust/scheduler/src/scheduler_server/grpc.rs
index 1ec73c04..af788dc4 100644
--- a/ballista/rust/scheduler/src/scheduler_server/grpc.rs
+++ b/ballista/rust/scheduler/src/scheduler_server/grpc.rs
@@ -22,10 +22,11 @@ use ballista_core::serde::protobuf::execute_query_params::{OptionalSessionId, Qu
 use ballista_core::serde::protobuf::executor_registration::OptionalHost;
 use ballista_core::serde::protobuf::scheduler_grpc_server::SchedulerGrpc;
 use ballista_core::serde::protobuf::{
-    ExecuteQueryParams, ExecuteQueryResult, ExecutorHeartbeat, GetFileMetadataParams,
-    GetFileMetadataResult, GetJobStatusParams, GetJobStatusResult, HeartBeatParams,
-    HeartBeatResult, PollWorkParams, PollWorkResult, RegisterExecutorParams,
-    RegisterExecutorResult, UpdateTaskStatusParams, UpdateTaskStatusResult,
+    CancelJobParams, CancelJobResult, ExecuteQueryParams, ExecuteQueryResult,
+    ExecutorHeartbeat, GetFileMetadataParams, GetFileMetadataResult, GetJobStatusParams,
+    GetJobStatusResult, HeartBeatParams, HeartBeatResult, PollWorkParams, PollWorkResult,
+    RegisterExecutorParams, RegisterExecutorResult, UpdateTaskStatusParams,
+    UpdateTaskStatusResult,
 };
 use ballista_core::serde::scheduler::{ExecutorData, ExecutorMetadata};
 use ballista_core::serde::AsExecutionPlan;
@@ -489,6 +490,26 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerGrpc
             }
         }
     }
+
+    async fn cancel_job(
+        &self,
+        request: Request<CancelJobParams>,
+    ) -> Result<Response<CancelJobResult>, Status> {
+        let job_id = request.into_inner().job_id;
+        info!("Received cancellation request for job {}", job_id);
+
+        self.state
+            .task_manager
+            .cancel_job(&job_id, &self.state.executor_manager)
+            .await
+            .map_err(|e| {
+                let msg = format!("Error cancelling job {}: {:?}", job_id, e);
+                error!("{}", msg);
+                Status::internal(msg)
+            })?;
+
+        Ok(Response::new(CancelJobResult { cancelled: true }))
+    }
 }
 
 #[cfg(all(test, feature = "sled"))]
diff --git a/ballista/rust/scheduler/src/scheduler_server/mod.rs b/ballista/rust/scheduler/src/scheduler_server/mod.rs
index e84f595a..a672c597 100644
--- a/ballista/rust/scheduler/src/scheduler_server/mod.rs
+++ b/ballista/rust/scheduler/src/scheduler_server/mod.rs
@@ -34,6 +34,7 @@ use crate::scheduler_server::event::{QueryStageSchedulerEvent, SchedulerServerEv
 use crate::scheduler_server::event_loop::SchedulerServerEventAction;
 use crate::scheduler_server::query_stage_scheduler::QueryStageScheduler;
 use crate::state::backend::StateBackendClient;
+use crate::state::executor_manager::ExecutorReservation;
 use crate::state::SchedulerState;
 
 // include the generated protobuf source as a submodule
@@ -225,7 +226,19 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> SchedulerServer<T
                     "Failed to update {} task statuses for executor {}: {:?}",
                     num_status, executor_id, e
                 );
-                // TODO what do we do here?
+                // In case task update fails, make sure to free reservations
+                if let Some(event_loop) = self.event_loop.as_ref() {
+                    let mut reservations = vec![];
+                    for _ in 0..num_status {
+                        reservations
+                            .push(ExecutorReservation::new_free(executor_id.to_owned()));
+                    }
+
+                    event_loop
+                        .get_sender()?
+                        .post_event(SchedulerServerEvent::Offer(reservations))
+                        .await?;
+                }
             }
         }
 
diff --git a/ballista/rust/scheduler/src/state/execution_graph.rs b/ballista/rust/scheduler/src/state/execution_graph.rs
index e3599092..b831ffe9 100644
--- a/ballista/rust/scheduler/src/state/execution_graph.rs
+++ b/ballista/rust/scheduler/src/state/execution_graph.rs
@@ -317,6 +317,29 @@ impl ExecutionGraph {
             .sum()
     }
 
+    /// Return all currently running tasks along with the executor ID on which they are assigned
+    pub fn running_tasks(&self) -> Vec<(PartitionId, String)> {
+        self.stages
+            .iter()
+            .flat_map(|(_, stage)| {
+                stage
+                    .running_tasks()
+                    .iter()
+                    .map(|(stage_id, partition, executor_id)| {
+                        (
+                            PartitionId {
+                                job_id: self.job_id.clone(),
+                                stage_id: *stage_id,
+                                partition_id: *partition,
+                            },
+                            executor_id.clone(),
+                        )
+                    })
+                    .collect::<Vec<(PartitionId, String)>>()
+            })
+            .collect::<Vec<(PartitionId, String)>>()
+    }
+
     /// Get next task that can be assigned to the given executor.
     /// This method should only be called when the resulting task is immediately
     /// being launched as the status will be set to Running and it will not be
@@ -748,6 +771,24 @@ impl ExecutionStage {
         }
     }
 
+    /// Returns a vector of currently running tasks in this stage
+    pub fn running_tasks(&self) -> Vec<(usize, usize, String)> {
+        if self.resolved {
+            self.task_statuses
+                .iter()
+                .enumerate()
+                .filter_map(|(partition, status)| match status {
+                    Some(task_status::Status::Running(RunningTask { executor_id })) => {
+                        Some((self.stage_id, partition, executor_id.clone()))
+                    }
+                    _ => None,
+                })
+                .collect()
+        } else {
+            vec![]
+        }
+    }
+
     /// Returns true if all inputs are complete and we can resolve all
     /// UnresolvedShuffleExec operators to ShuffleReadExec
     pub fn resolvable(&self) -> bool {
diff --git a/ballista/rust/scheduler/src/state/executor_manager.rs b/ballista/rust/scheduler/src/state/executor_manager.rs
index ad2a4389..4c5449c8 100644
--- a/ballista/rust/scheduler/src/state/executor_manager.rs
+++ b/ballista/rust/scheduler/src/state/executor_manager.rs
@@ -23,12 +23,17 @@ use crate::state::{decode_into, decode_protobuf, encode_protobuf, with_lock};
 use ballista_core::error::{BallistaError, Result};
 use ballista_core::serde::protobuf;
 
+use ballista_core::serde::protobuf::executor_grpc_client::ExecutorGrpcClient;
 use ballista_core::serde::scheduler::{ExecutorData, ExecutorMetadata};
+use ballista_core::utils::create_grpc_client_connection;
 use futures::StreamExt;
 use log::{debug, info};
 use parking_lot::RwLock;
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
+use tonic::transport::Channel;
+
+type ExecutorClients = Arc<RwLock<HashMap<String, ExecutorGrpcClient<Channel>>>>;
 
 /// Represents a task slot that is reserved (i.e. available for scheduling but not visible to the
 /// rest of the system).
@@ -71,6 +76,7 @@ pub(crate) struct ExecutorManager {
     state: Arc<dyn StateBackendClient>,
     executor_metadata: Arc<RwLock<HashMap<String, ExecutorMetadata>>>,
     executors_heartbeat: Arc<RwLock<HashMap<String, protobuf::ExecutorHeartbeat>>>,
+    clients: ExecutorClients,
 }
 
 impl ExecutorManager {
@@ -79,6 +85,7 @@ impl ExecutorManager {
             state,
             executor_metadata: Arc::new(RwLock::new(HashMap::new())),
             executors_heartbeat: Arc::new(RwLock::new(HashMap::new())),
+            clients: Default::default(),
         }
     }
 
@@ -194,6 +201,34 @@ impl ExecutorManager {
         .await
     }
 
+    pub async fn get_client(
+        &self,
+        executor_id: &str,
+    ) -> Result<ExecutorGrpcClient<Channel>> {
+        let client = {
+            let clients = self.clients.read();
+            clients.get(executor_id).cloned()
+        };
+
+        if let Some(client) = client {
+            Ok(client)
+        } else {
+            let executor_metadata = self.get_executor_metadata(executor_id).await?;
+            let executor_url = format!(
+                "http://{}:{}",
+                executor_metadata.host, executor_metadata.grpc_port
+            );
+            let connection = create_grpc_client_connection(executor_url).await?;
+            let client = ExecutorGrpcClient::new(connection);
+
+            {
+                let mut clients = self.clients.write();
+                clients.insert(executor_id.to_owned(), client.clone());
+            }
+            Ok(client)
+        }
+    }
+
     /// Get a list of all executors along with the timestamp of their last recorded heartbeat
     pub async fn get_executor_state(&self) -> Result<Vec<(ExecutorMetadata, Duration)>> {
         let heartbeat_timestamps: Vec<(String, u64)> = {
diff --git a/ballista/rust/scheduler/src/state/task_manager.rs b/ballista/rust/scheduler/src/state/task_manager.rs
index bb0db021..58c49a3b 100644
--- a/ballista/rust/scheduler/src/state/task_manager.rs
+++ b/ballista/rust/scheduler/src/state/task_manager.rs
@@ -17,9 +17,9 @@
 
 use crate::scheduler_server::event::QueryStageSchedulerEvent;
 use crate::scheduler_server::SessionBuilder;
-use crate::state::backend::{Keyspace, StateBackendClient};
+use crate::state::backend::{Keyspace, Lock, StateBackendClient};
 use crate::state::execution_graph::{ExecutionGraph, Task};
-use crate::state::executor_manager::ExecutorReservation;
+use crate::state::executor_manager::{ExecutorManager, ExecutorReservation};
 use crate::state::{decode_protobuf, encode_protobuf, with_lock};
 use ballista_core::config::BallistaConfig;
 #[cfg(not(test))]
@@ -29,7 +29,7 @@ use ballista_core::serde::protobuf::executor_grpc_client::ExecutorGrpcClient;
 
 use crate::state::session_manager::create_datafusion_context;
 use ballista_core::serde::protobuf::{
-    self, job_status, FailedJob, JobStatus, PartitionId, TaskDefinition, TaskStatus,
+    self, job_status, CancelTasksParams, FailedJob, JobStatus, TaskDefinition, TaskStatus,
 };
 use ballista_core::serde::scheduler::to_proto::hash_partitioning_to_proto;
 use ballista_core::serde::scheduler::ExecutorMetadata;
@@ -37,7 +37,7 @@ use ballista_core::serde::{AsExecutionPlan, BallistaCodec};
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion::prelude::SessionContext;
 use datafusion_proto::logical_plan::AsLogicalPlan;
-use log::{debug, info, warn};
+use log::{debug, error, info, warn};
 use rand::distributions::Alphanumeric;
 use rand::{thread_rng, Rng};
 use std::collections::{HashMap, HashSet};
@@ -79,6 +79,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
         plan: Arc<dyn ExecutionPlan>,
     ) -> Result<()> {
         let graph = ExecutionGraph::new(job_id, session_id, plan)?;
+        info!("Submitting execution graph: {:?}", graph);
         self.state
             .put(
                 Keyspace::ActiveJobs,
@@ -283,7 +284,6 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
                                     reservation.executor_id, task
                                 );
                                 assignments.push((executor_id.clone(), task));
-                                // locks.push(lock);
                                 graphs.insert(job_id, graph);
                                 assigned = true;
                                 break;
@@ -333,11 +333,78 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
         .await
     }
 
+    pub(crate) async fn cancel_job(
+        &self,
+        job_id: &str,
+        executor_manager: &ExecutorManager,
+    ) -> Result<()> {
+        let lock = self.state.lock(Keyspace::ActiveJobs, "").await?;
+
+        let running_tasks = self
+            .get_execution_graph(job_id)
+            .await
+            .map(|graph| graph.running_tasks())
+            .unwrap_or_else(|_| vec![]);
+
+        info!(
+            "Cancelling {} running tasks for job {}",
+            running_tasks.len(),
+            job_id
+        );
+
+        self.fail_job_inner(lock, job_id, "Cancelled".to_owned())
+            .await?;
+
+        let mut tasks: HashMap<&str, Vec<protobuf::PartitionId>> = Default::default();
+
+        for (partition, executor_id) in &running_tasks {
+            if let Some(parts) = tasks.get_mut(executor_id.as_str()) {
+                parts.push(protobuf::PartitionId {
+                    job_id: job_id.to_owned(),
+                    stage_id: partition.stage_id as u32,
+                    partition_id: partition.partition_id as u32,
+                })
+            } else {
+                tasks.insert(
+                    executor_id.as_str(),
+                    vec![protobuf::PartitionId {
+                        job_id: job_id.to_owned(),
+                        stage_id: partition.stage_id as u32,
+                        partition_id: partition.partition_id as u32,
+                    }],
+                );
+            }
+        }
+
+        for (executor_id, partitions) in tasks {
+            if let Ok(mut client) = executor_manager.get_client(executor_id).await {
+                client
+                    .cancel_tasks(CancelTasksParams {
+                        partition_id: partitions,
+                    })
+                    .await?;
+            } else {
+                error!("Failed to get client for executor ID {}", executor_id)
+            }
+        }
+
+        Ok(())
+    }
+
     /// Mark a job as failed. This will create a key under the FailedJobs keyspace
     /// and remove the job from ActiveJobs or QueuedJobs
     /// TODO this should be atomic
     pub async fn fail_job(&self, job_id: &str, error_message: String) -> Result<()> {
         let lock = self.state.lock(Keyspace::ActiveJobs, "").await?;
+        self.fail_job_inner(lock, job_id, error_message).await
+    }
+
+    async fn fail_job_inner(
+        &self,
+        lock: Box<dyn Lock>,
+        job_id: &str,
+        error_message: String,
+    ) -> Result<()> {
         with_lock(lock, self.state.delete(Keyspace::ActiveJobs, job_id)).await?;
 
         let status = JobStatus {
@@ -354,7 +421,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
 
     #[cfg(not(test))]
     /// Launch the given task on the specified executor
-    pub async fn launch_task(
+    pub(crate) async fn launch_task(
         &self,
         executor: &ExecutorMetadata,
         task: Task,
@@ -398,7 +465,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
 
     /// In unit tests, we do not have actual executors running, so it simplifies things to just noop.
     #[cfg(test)]
-    pub async fn launch_task(
+    pub(crate) async fn launch_task(
         &self,
         _executor: &ExecutorMetadata,
         _task: Task,
@@ -426,7 +493,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskManager<T, U>
             hash_partitioning_to_proto(task.output_partitioning.as_ref())?;
 
         let task_definition = TaskDefinition {
-            task_id: Some(PartitionId {
+            task_id: Some(protobuf::PartitionId {
                 job_id: task.partition.job_id.clone(),
                 stage_id: task.partition.stage_id as u32,
                 partition_id: task.partition.partition_id as u32,