You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mx...@apache.org on 2016/08/29 15:33:27 UTC

[7/9] flink git commit: [FLINK-1984] Mesos ResourceManager - T1 milstone (3)

[FLINK-1984] Mesos ResourceManager - T1 milstone (3)

- Fenzo usage fix - always call scheduleOnce after expireAllLeases.
- increased aggressiveness of task scheduler
- factored YarnJobManager and MesosJobManager to share base class
`ContaineredJobManager`
- improved supervision for task actors, unit tests
- support for zombie tasks (i.e. non-strict slave registry)
- improved javadocs
- fix for style violations (e.g. line length)
- completed the SchedulerProxy
- final fields
- improved preconditions
- log lines to use {}
- cleanup ZK state
- serializable messages


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/59eeea4c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/59eeea4c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/59eeea4c

Branch: refs/heads/master
Commit: 59eeea4ce4fb0361c2f02618f5f4dbcfc6c0467d
Parents: 67be9e0
Author: wrighe3 <er...@emc.com>
Authored: Wed Aug 24 09:27:26 2016 -0700
Committer: Maximilian Michels <mx...@apache.org>
Committed: Mon Aug 29 17:32:51 2016 +0200

----------------------------------------------------------------------
 .../main/java/org/apache/flink/mesos/Utils.java |  13 +-
 .../flink/mesos/cli/FlinkMesosSessionCli.java   |  13 ++
 .../clusterframework/LaunchableMesosWorker.java |   9 +-
 .../MesosApplicationMasterRunner.java           |  61 ++----
 .../MesosFlinkResourceManager.java              |  43 ++--
 .../MesosTaskManagerParameters.java             |  12 ++
 .../MesosTaskManagerRunner.java                 |  10 +-
 .../store/MesosWorkerStore.java                 | 119 ++++++++---
 .../store/StandaloneMesosWorkerStore.java       |  11 +-
 .../store/ZooKeeperMesosWorkerStore.java        |  52 +++--
 .../flink/mesos/scheduler/SchedulerProxy.java   |  10 +-
 .../mesos/scheduler/TaskSchedulerBuilder.java   |   7 +
 .../mesos/scheduler/messages/AcceptOffers.java  |  13 +-
 .../mesos/scheduler/messages/Connected.java     |   5 +-
 .../mesos/scheduler/messages/Disconnected.java  |   7 +-
 .../flink/mesos/scheduler/messages/Error.java   |   9 +-
 .../mesos/scheduler/messages/ExecutorLost.java  |  66 ++++++
 .../scheduler/messages/FrameworkMessage.java    |  67 ++++++
 .../scheduler/messages/OfferRescinded.java      |   9 +-
 .../mesos/scheduler/messages/ReRegistered.java  |   5 +-
 .../mesos/scheduler/messages/Registered.java    |   6 +-
 .../scheduler/messages/ResourceOffers.java      |   8 +-
 .../mesos/scheduler/messages/SlaveLost.java     |   9 +-
 .../mesos/scheduler/messages/StatusUpdate.java  |   9 +-
 .../flink/mesos/util/MesosArtifactServer.java   |  12 +-
 .../flink/mesos/util/MesosConfiguration.java    |  23 +-
 .../apache/flink/mesos/util/ZooKeeperUtils.java |   2 +-
 .../clusterframework/MesosTaskManager.scala     |  16 +-
 .../mesos/scheduler/LaunchCoordinator.scala     |  63 +++---
 .../scheduler/ReconciliationCoordinator.scala   |  10 +-
 .../flink/mesos/scheduler/TaskMonitor.scala     |  16 +-
 .../apache/flink/mesos/scheduler/Tasks.scala    |  67 +++---
 .../ContaineredJobManager.scala                 | 174 ---------------
 .../MesosFlinkResourceManagerTest.java          |  29 +--
 .../scala/org/apache/flink/mesos/Utils.scala    |   6 +-
 .../mesos/scheduler/LaunchCoordinatorTest.scala |  91 +++++---
 .../ReconciliationCoordinatorTest.scala         |   3 +-
 .../flink/mesos/scheduler/TaskMonitorTest.scala |  14 +-
 .../flink/mesos/scheduler/TasksTest.scala       | 210 +++++++++++++++++++
 .../org/apache/flink/runtime/akka/FSMSpec.scala |   3 +-
 .../ContaineredJobManager.scala                 | 172 +++++++++++++++
 .../flink/yarn/YarnApplicationMasterRunner.java |  16 +-
 .../org/apache/flink/yarn/YarnJobManager.scala  | 104 +--------
 43 files changed, 1040 insertions(+), 564 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/Utils.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/Utils.java b/flink-mesos/src/main/java/org/apache/flink/mesos/Utils.java
index bd5ef50..173ae33 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/Utils.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/Utils.java
@@ -26,7 +26,7 @@ import java.util.Arrays;
 public class Utils {
 	/**
 	 * Construct a Mesos environment variable.
-     */
+	 */
 	public static Protos.Environment.Variable variable(String name, String value) {
 		return Protos.Environment.Variable.newBuilder()
 			.setName(name)
@@ -36,7 +36,7 @@ public class Utils {
 
 	/**
 	 * Construct a Mesos URI.
-     */
+	 */
 	public static Protos.CommandInfo.URI uri(URL url, boolean cacheable) {
 		return Protos.CommandInfo.URI.newBuilder()
 			.setValue(url.toExternalForm())
@@ -45,6 +45,9 @@ public class Utils {
 			.build();
 	}
 
+	/**
+	 * Construct a scalar resource value.
+	 */
 	public static Protos.Resource scalar(String name, double value) {
 		return Protos.Resource.newBuilder()
 			.setName(name)
@@ -53,10 +56,16 @@ public class Utils {
 			.build();
 	}
 
+	/**
+	 * Construct a range value.
+	 */
 	public static Protos.Value.Range range(long begin, long end) {
 		return Protos.Value.Range.newBuilder().setBegin(begin).setEnd(end).build();
 	}
 
+	/**
+	 * Construct a ranges resource value.
+	 */
 	public static Protos.Resource ranges(String name, Protos.Value.Range... ranges) {
 		return Protos.Resource.newBuilder()
 			.setName(name)

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/cli/FlinkMesosSessionCli.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/cli/FlinkMesosSessionCli.java b/flink-mesos/src/main/java/org/apache/flink/mesos/cli/FlinkMesosSessionCli.java
index 278f1a2..dcce0b8 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/cli/FlinkMesosSessionCli.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/cli/FlinkMesosSessionCli.java
@@ -26,10 +26,18 @@ import org.apache.flink.configuration.Configuration;
 import java.io.IOException;
 import java.util.Map;
 
+/**
+ * Class handling the command line interface to the Mesos session.
+ */
 public class FlinkMesosSessionCli {
 
 	private static final ObjectMapper mapper = new ObjectMapper();
 
+	/**
+	 * Decode encoded dynamic properties.
+	 * @param dynamicPropertiesEncoded encoded properties produced by the encoding method.
+	 * @return a configuration instance to be merged with the static configuration.
+	 */
 	public static Configuration decodeDynamicProperties(String dynamicPropertiesEncoded) {
 		try {
 			Configuration configuration = new Configuration();
@@ -47,6 +55,11 @@ public class FlinkMesosSessionCli {
 		}
 	}
 
+	/**
+	 * Encode dynamic properties as a string to be transported as an environment variable.
+	 * @param configuration the dynamic properties to encode.
+	 * @return a string to be decoded later.
+	 */
 	public static String encodeDynamicProperties(Configuration configuration) {
 		try {
 			String dynamicPropertiesEncoded = mapper.writeValueAsString(configuration.toMap());

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
index d416ec1..5f940b5 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/LaunchableMesosWorker.java
@@ -59,7 +59,7 @@ public class LaunchableMesosWorker implements LaunchableTask {
 	 * @param params the TM parameters such as memory, cpu to acquire.
 	 * @param template a template for the TaskInfo to be constructed at launch time.
 	 * @param taskID the taskID for this worker.
-     */
+	 */
 	public LaunchableMesosWorker(MesosTaskManagerParameters params, Protos.TaskInfo.Builder template, Protos.TaskID taskID) {
 		this.params = params;
 		this.template = template;
@@ -152,8 +152,8 @@ public class LaunchableMesosWorker implements LaunchableTask {
 	 * Construct the TaskInfo needed to launch the worker.
 	 * @param slaveId the assigned slave.
 	 * @param assignment the assignment details.
-     * @return a fully-baked TaskInfo.
-     */
+	 * @return a fully-baked TaskInfo.
+	 */
 	@Override
 	public Protos.TaskInfo launch(Protos.SlaveID slaveId, TaskAssignmentResult assignment) {
 
@@ -167,10 +167,9 @@ public class LaunchableMesosWorker implements LaunchableTask {
 			.setName(taskID.getValue())
 			.addResources(scalar("cpus", assignment.getRequest().getCPUs()))
 			.addResources(scalar("mem", assignment.getRequest().getMemory()));
-		//.addResources(scalar("disk", assignment.getRequest.getDisk).setRole("Flink"))
 
 		// use the assigned ports for the TM
-		if (assignment.getAssignedPorts().size() != TM_PORT_KEYS.length) {
+		if (assignment.getAssignedPorts().size() < TM_PORT_KEYS.length) {
 			throw new IllegalArgumentException("unsufficient # of ports assigned");
 		}
 		for (int i = 0; i < TM_PORT_KEYS.length; i++) {

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
index 1bd02c4..c13cdf9 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
@@ -20,6 +20,7 @@ package org.apache.flink.mesos.runtime.clusterframework;
 
 import akka.actor.ActorRef;
 import akka.actor.ActorSystem;
+import akka.actor.Address;
 import akka.actor.Props;
 
 import org.apache.curator.framework.CuratorFramework;
@@ -69,6 +70,8 @@ import java.util.concurrent.TimeUnit;
 import static org.apache.flink.mesos.Utils.uri;
 import static org.apache.flink.mesos.Utils.variable;
 
+import static org.apache.flink.util.Preconditions.checkState;
+
 /**
  * This class is the executable entry point for the Mesos Application Master.
  * It starts actor system and the actors for {@link org.apache.flink.runtime.jobmanager.JobManager}
@@ -170,10 +173,10 @@ public class MesosApplicationMasterRunner {
 			// configuration problem occurs
 
 			final String workingDir = ENV.get(MesosConfigKeys.ENV_MESOS_SANDBOX);
-			require(workingDir != null, "Sandbox directory variable (%s) not set", MesosConfigKeys.ENV_MESOS_SANDBOX);
+			checkState(workingDir != null, "Sandbox directory variable (%s) not set", MesosConfigKeys.ENV_MESOS_SANDBOX);
 
 			final String sessionID = ENV.get(MesosConfigKeys.ENV_SESSION_ID);
-			require(sessionID != null, "Session ID (%s) not set", MesosConfigKeys.ENV_SESSION_ID);
+			checkState(sessionID != null, "Session ID (%s) not set", MesosConfigKeys.ENV_SESSION_ID);
 
 			// Note that we use the "appMasterHostname" given by the system, to make sure
 			// we use the hostnames consistently throughout akka.
@@ -232,7 +235,7 @@ public class MesosApplicationMasterRunner {
 			// JM endpoint, which should be explicitly configured by the dispatcher (based on acquired net resources)
 			final int listeningPort = config.getInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY,
 				ConfigConstants.DEFAULT_JOB_MANAGER_IPC_PORT);
-			require(listeningPort >= 0 && listeningPort <= 65536, "Config parameter \"" +
+			checkState(listeningPort >= 0 && listeningPort <= 65536, "Config parameter \"" +
 				ConfigConstants.JOB_MANAGER_IPC_PORT_KEY + "\" is invalid, it must be between 0 and 65536");
 
 			// ----------------- (2) start the actor system -------------------
@@ -241,8 +244,9 @@ public class MesosApplicationMasterRunner {
 			// using the configured address and ports
 			actorSystem = BootstrapTools.startActorSystem(config, appMasterHostname, listeningPort, LOG);
 
-			final String akkaHostname = AkkaUtils.getAddress(actorSystem).host().get();
-			final int akkaPort = (Integer) AkkaUtils.getAddress(actorSystem).port().get();
+			Address address = AkkaUtils.getAddress(actorSystem);
+			final String akkaHostname = address.host().get();
+			final int akkaPort = (Integer) address.port().get();
 
 			LOG.info("Actor system bound to hostname {}.", akkaHostname);
 
@@ -259,7 +263,7 @@ public class MesosApplicationMasterRunner {
 			LOG.debug("TaskManager configuration: {}", taskManagerConfig);
 
 			final Protos.TaskInfo.Builder taskManagerContext = createTaskManagerContext(
-				config, mesosConfig, ENV,
+				config, ENV,
 				taskManagerParameters, taskManagerConfig,
 				workingDir, getTaskManagerClass(), artifactServer, LOG);
 
@@ -315,7 +319,6 @@ public class MesosApplicationMasterRunner {
 
 			ActorRef resourceMaster = actorSystem.actorOf(resourceMasterProps, "Mesos_Resource_Master");
 
-
 			// 4: Process reapers
 			// The process reapers ensure that upon unexpected actor death, the process exits
 			// and does not stay lingering around unresponsive
@@ -334,14 +337,6 @@ public class MesosApplicationMasterRunner {
 			// make sure that everything whatever ends up in the log
 			LOG.error("Mesos JobManager initialization failed", t);
 
-			if (actorSystem != null) {
-				try {
-					actorSystem.shutdown();
-				} catch (Throwable tt) {
-					LOG.error("Error shutting down actor system", tt);
-				}
-			}
-
 			if (webMonitor != null) {
 				try {
 					webMonitor.stop();
@@ -358,6 +353,14 @@ public class MesosApplicationMasterRunner {
 				}
 			}
 
+			if (actorSystem != null) {
+				try {
+					actorSystem.shutdown();
+				} catch (Throwable tt) {
+					LOG.error("Error shutting down actor system", tt);
+				}
+			}
+
 			return INIT_ERROR_EXIT_CODE;
 		}
 
@@ -407,29 +410,14 @@ public class MesosApplicationMasterRunner {
 	}
 
 	/**
-	 * Validates a condition, throwing a RuntimeException if the condition is violated.
-	 *
-	 * @param condition The condition.
-	 * @param message The message for the runtime exception, with format variables as defined by
-	 *                {@link String#format(String, Object...)}.
-	 * @param values The format arguments.
-	 */
-	private static void require(boolean condition, String message, Object... values) {
-		if (!condition) {
-			throw new RuntimeException(String.format(message, values));
-		}
-	}
-
-	/**
 	 *
 	 * @param baseDirectory
 	 * @param additional
 	 *
 	 * @return The configuration to be used by the TaskManagers.
 	 */
-	@SuppressWarnings("deprecation")
 	private static Configuration createConfiguration(String baseDirectory, Configuration additional) {
-		LOG.info("Loading config from directory " + baseDirectory);
+		LOG.info("Loading config from directory {}", baseDirectory);
 
 		Configuration configuration = GlobalConfiguration.loadConfiguration(baseDirectory);
 
@@ -491,7 +479,7 @@ public class MesosApplicationMasterRunner {
 		return mesos;
 	}
 
-	private MesosWorkerStore createWorkerStore(Configuration flinkConfig) throws Exception {
+	private static MesosWorkerStore createWorkerStore(Configuration flinkConfig) throws Exception {
 		MesosWorkerStore workerStore;
 		RecoveryMode recoveryMode = RecoveryMode.fromConfig(flinkConfig);
 		if (recoveryMode == RecoveryMode.STANDALONE) {
@@ -523,8 +511,6 @@ public class MesosApplicationMasterRunner {
 	 *
 	 * @param flinkConfig
 	 *         The Flink configuration object.
-	 * @param mesosConfig
-	 *         The Mesos configuration object.
 	 * @param env
 	 *         The environment variables.
 	 * @param tmParams
@@ -547,7 +533,6 @@ public class MesosApplicationMasterRunner {
 	 */
 	public static Protos.TaskInfo.Builder createTaskManagerContext(
 		Configuration flinkConfig,
-		MesosConfiguration mesosConfig,
 		Map<String, String> env,
 		MesosTaskManagerParameters tmParams,
 		Configuration taskManagerConfig,
@@ -563,13 +548,13 @@ public class MesosApplicationMasterRunner {
 		log.info("Setting up artifacts for TaskManagers");
 
 		String shipListString = env.get(MesosConfigKeys.ENV_CLIENT_SHIP_FILES);
-		require(shipListString != null, "Environment variable %s not set", MesosConfigKeys.ENV_CLIENT_SHIP_FILES);
+		checkState(shipListString != null, "Environment variable %s not set", MesosConfigKeys.ENV_CLIENT_SHIP_FILES);
 
 		String clientUsername = env.get(MesosConfigKeys.ENV_CLIENT_USERNAME);
-		require(clientUsername != null, "Environment variable %s not set", MesosConfigKeys.ENV_CLIENT_USERNAME);
+		checkState(clientUsername != null, "Environment variable %s not set", MesosConfigKeys.ENV_CLIENT_USERNAME);
 
 		String classPathString = env.get(MesosConfigKeys.ENV_FLINK_CLASSPATH);
-		require(classPathString != null, "Environment variable %s not set", MesosConfigKeys.ENV_FLINK_CLASSPATH);
+		checkState(classPathString != null, "Environment variable %s not set", MesosConfigKeys.ENV_FLINK_CLASSPATH);
 
 		// register the Flink jar
 		final File flinkJarFile = new File(workingDirectory, "flink.jar");

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManager.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManager.java
index 66fff7d..6b24ee8 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManager.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManager.java
@@ -96,8 +96,9 @@ public class MesosFlinkResourceManager extends FlinkResourceManager<RegisteredMe
 
 	private ActorRef reconciliationCoordinator;
 
-	private MesosWorkerStore workerStore;
+	private final MesosWorkerStore workerStore;
 
+	/** planning state related to workers - package private for unit test purposes */
 	final Map<ResourceID, MesosWorkerStore.Worker> workersInNew;
 	final Map<ResourceID, MesosWorkerStore.Worker> workersInLaunch;
 	final Map<ResourceID, MesosWorkerStore.Worker> workersBeingReturned;
@@ -250,7 +251,7 @@ public class MesosFlinkResourceManager extends FlinkResourceManager<RegisteredMe
 	 *
 	 * @param finalStatus The application status to report.
 	 * @param optionalDiagnostics An optional diagnostics message.
-     */
+	 */
 	@Override
 	protected void shutdownApplication(ApplicationStatus finalStatus, String optionalDiagnostics) {
 
@@ -264,10 +265,10 @@ public class MesosFlinkResourceManager extends FlinkResourceManager<RegisteredMe
 		}
 
 		try {
-			workerStore.cleanup();
+			workerStore.stop(true);
 		}
 		catch(Exception ex) {
-			LOG.warn("unable to cleanup the ZooKeeper state", ex);
+			LOG.warn("unable to stop the worker state store", ex);
 		}
 
 		context().stop(self());
@@ -336,7 +337,7 @@ public class MesosFlinkResourceManager extends FlinkResourceManager<RegisteredMe
 	 * Plan for some additional workers to be launched.
 	 *
 	 * @param numWorkers The number of workers to allocate.
-     */
+	 */
 	@Override
 	protected void requestNewWorkers(int numWorkers) {
 
@@ -346,7 +347,7 @@ public class MesosFlinkResourceManager extends FlinkResourceManager<RegisteredMe
 
 			// generate new workers into persistent state and launch associated actors
 			for (int i = 0; i < numWorkers; i++) {
-				MesosWorkerStore.Worker worker = MesosWorkerStore.Worker.newTask(workerStore.newTaskID());
+				MesosWorkerStore.Worker worker = MesosWorkerStore.Worker.newWorker(workerStore.newTaskID());
 				workerStore.putWorker(worker);
 				workersInNew.put(extractResourceID(worker.taskID()), worker);
 
@@ -379,7 +380,7 @@ public class MesosFlinkResourceManager extends FlinkResourceManager<RegisteredMe
 	 *
 	 * Acceptance is routed through the RM to update the persistent state before
 	 * forwarding the message to Mesos.
-     */
+	 */
 	private void acceptOffers(AcceptOffers msg) {
 
 		try {
@@ -394,7 +395,7 @@ public class MesosFlinkResourceManager extends FlinkResourceManager<RegisteredMe
 					MesosWorkerStore.Worker worker = workersInNew.remove(extractResourceID(info.getTaskId()));
 					assert (worker != null);
 
-					worker = worker.launchTask(info.getSlaveId(), msg.hostname());
+					worker = worker.launchWorker(info.getSlaveId(), msg.hostname());
 					workerStore.putWorker(worker);
 					workersInLaunch.put(extractResourceID(worker.taskID()), worker);
 
@@ -450,7 +451,7 @@ public class MesosFlinkResourceManager extends FlinkResourceManager<RegisteredMe
 	 *
 	 * @param toConsolidate The worker IDs known previously to the JobManager.
 	 * @return A collection of registered worker node records.
-     */
+	 */
 	@Override
 	protected Collection<RegisteredMesosWorkerNode> reacceptRegisteredWorkers(Collection<ResourceID> toConsolidate) {
 
@@ -497,13 +498,13 @@ public class MesosFlinkResourceManager extends FlinkResourceManager<RegisteredMe
 
 	/**
 	 * Plan for the removal of the given worker.
-     */
+	 */
 	private void releaseWorker(MesosWorkerStore.Worker worker) {
 		try {
 			LOG.info("Releasing worker {}", worker.taskID());
 
 			// update persistent state of worker to Released
-			worker = worker.releaseTask();
+			worker = worker.releaseWorker();
 			workerStore.putWorker(worker);
 			workersBeingReturned.put(extractResourceID(worker.taskID()), worker);
 			taskRouter.tell(new TaskMonitor.TaskGoalStateUpdated(extractGoalState(worker)), self());
@@ -580,20 +581,26 @@ public class MesosFlinkResourceManager extends FlinkResourceManager<RegisteredMe
 
 	/**
 	 * Invoked when a Mesos task reaches a terminal status.
-     */
+	 */
 	private void taskTerminated(Protos.TaskID taskID, Protos.TaskStatus status) {
 		// this callback occurs for failed containers and for released containers alike
 
 		final ResourceID id = extractResourceID(taskID);
 
+		boolean existed;
 		try {
-			workerStore.removeWorker(taskID);
+			existed = workerStore.removeWorker(taskID);
 		}
 		catch(Exception ex) {
 			fatalError("unable to remove worker", ex);
 			return;
 		}
 
+		if(!existed) {
+			LOG.info("Received a termination notice for an unrecognized worker: {}", id);
+			return;
+		}
+
 		// check if this is a failed task or a released task
 		if (workersBeingReturned.remove(id) != null) {
 			// regular finished worker that we released
@@ -624,7 +631,7 @@ public class MesosFlinkResourceManager extends FlinkResourceManager<RegisteredMe
 			sendInfoMessage(diagMessage);
 
 			LOG.info(diagMessage);
-			LOG.info("Total number of failed tasks so far: " + failedTasksSoFar);
+			LOG.info("Total number of failed tasks so far: {}", failedTasksSoFar);
 
 			// maxFailedTasks == -1 is infinite number of retries.
 			if (maxFailedTasks >= 0 && failedTasksSoFar > maxFailedTasks) {
@@ -672,20 +679,20 @@ public class MesosFlinkResourceManager extends FlinkResourceManager<RegisteredMe
 	 * Extracts the Mesos task goal state from the worker information.
 	 * @param worker the persistent worker information.
 	 * @return goal state information for the {@Link TaskMonitor}.
-     */
+	 */
 	static TaskMonitor.TaskGoalState extractGoalState(MesosWorkerStore.Worker worker) {
 		switch(worker.state()) {
 			case New: return new TaskMonitor.New(worker.taskID());
 			case Launched: return new TaskMonitor.Launched(worker.taskID(), worker.slaveID().get());
 			case Released: return new TaskMonitor.Released(worker.taskID(), worker.slaveID().get());
-			default: throw new IllegalArgumentException();
+			default: throw new IllegalArgumentException("unsupported worker state");
 		}
 	}
 
 	/**
 	 * Creates the Fenzo optimizer (builder).
-	 * The builder is an indirection to faciliate unit testing of the Launch Coordinator.
-     */
+	 * The builder is an indirection to facilitate unit testing of the Launch Coordinator.
+	 */
 	private static TaskSchedulerBuilder createOptimizer() {
 		return new TaskSchedulerBuilder() {
 			TaskScheduler.Builder builder = new TaskScheduler.Builder();

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
index 5321ab6..1b19d08 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java
@@ -24,6 +24,12 @@ import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameter
 
 import static java.util.Objects.requireNonNull;
 
+/**
+ * This class describes the Mesos-specific parameters for launching a TaskManager process.
+ *
+ * These parameters are in addition to the common parameters
+ * provided by {@link ContaineredTaskManagerParameters}.
+ */
 public class MesosTaskManagerParameters {
 
 	private double cpus;
@@ -36,10 +42,16 @@ public class MesosTaskManagerParameters {
 		this.containeredParameters = containeredParameters;
 	}
 
+	/**
+	 * Get the CPU units to use for the TaskManager process.
+     */
 	public double cpus() {
 		return cpus;
 	}
 
+	/**
+	 * Get the common containered parameters.
+     */
 	public ContaineredTaskManagerParameters containeredParameters() {
 		return containeredParameters;
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerRunner.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerRunner.java
index e34def6..94a9e99 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerRunner.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerRunner.java
@@ -63,7 +63,7 @@ public class MesosTaskManagerRunner {
 			configuration.addAll(dynamicProperties);
 		}
 		catch (Throwable t) {
-			LOG.error(t.getMessage(), t);
+			LOG.error("Failed to load the TaskManager configuration and dynamic properties.", t);
 			System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
 			return;
 		}
@@ -77,15 +77,15 @@ public class MesosTaskManagerRunner {
 		String flinkTempDirs = configuration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, null);
 		if (flinkTempDirs != null) {
 			LOG.info("Overriding Mesos temporary file directories with those " +
-				"specified in the Flink config: " + flinkTempDirs);
+				"specified in the Flink config: {}", flinkTempDirs);
 		}
 		else if (tmpDirs != null) {
-			LOG.info("Setting directories for temporary files to: " + tmpDirs);
+			LOG.info("Setting directories for temporary files to: {}", tmpDirs);
 			configuration.setString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, tmpDirs);
 		}
 
-		LOG.info("Mesos task runs as '" + UserGroupInformation.getCurrentUser().getShortUserName() +
-			"', setting user to execute Flink TaskManager to '" + effectiveUsername + "'");
+		LOG.info("Mesos task runs as '{}', setting user to execute Flink TaskManager to '{}'",
+			UserGroupInformation.getCurrentUser().getShortUserName(), effectiveUsername);
 
 		// tell akka to die in case of an error
 		configuration.setBoolean(ConfigConstants.AKKA_JVM_EXIT_ON_FATAL_ERROR, true);

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/MesosWorkerStore.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/MesosWorkerStore.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/MesosWorkerStore.java
index 2671fc1..d6ff6bc 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/MesosWorkerStore.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/MesosWorkerStore.java
@@ -30,47 +30,75 @@ import static java.util.Objects.requireNonNull;
 
 /**
  * A store of Mesos workers and associated framework information.
- *
- * Generates a framework ID as necessary.
  */
 public interface MesosWorkerStore {
 
-	static final DecimalFormat TASKID_FORMAT = new DecimalFormat("taskmanager-00000");
+	/**
+	 * The template for naming the worker.
+	 */
+	DecimalFormat TASKID_FORMAT = new DecimalFormat("taskmanager-00000");
 
+	/**
+	 * Start the worker store.
+	 */
 	void start() throws Exception;
 
-	void stop() throws Exception;
+	/**
+	 * Stop the worker store.
+	 * @param cleanup if true, cleanup any stored information.
+	 */
+	void stop(boolean cleanup) throws Exception;
 
+	/**
+	 * Get the stored Mesos framework ID.
+	 */
 	Option<Protos.FrameworkID> getFrameworkID() throws Exception;
 
+	/**
+	 * Set the stored Mesos framework ID.
+	 */
 	void setFrameworkID(Option<Protos.FrameworkID> frameworkID) throws Exception;
 
+	/**
+	 * Recover the stored workers.
+	 */
 	List<Worker> recoverWorkers() throws Exception;
 
+	/**
+	 * Generate a new task ID for a worker.
+	 */
 	Protos.TaskID newTaskID() throws Exception;
 
+	/**
+	 * Put a worker into storage.
+	 */
 	void putWorker(Worker worker) throws Exception;
 
-	void removeWorker(Protos.TaskID taskID) throws Exception;
-
-	void cleanup() throws Exception;
+	/**
+	 * Remove a worker from storage.
+	 * @return true if the worker existed.
+	 */
+	boolean removeWorker(Protos.TaskID taskID) throws Exception;
 
 	/**
-	 * A stored task.
+	 * A stored worker.
 	 *
 	 * The assigned slaveid/hostname is valid in Launched and Released states.  The hostname is needed
 	 * by Fenzo for optimization purposes.
 	 */
 	class Worker implements Serializable {
-		private Protos.TaskID taskID;
 
-		private Option<Protos.SlaveID> slaveID;
+		private static final long serialVersionUID = 1L;
+
+		private final Protos.TaskID taskID;
 
-		private Option<String> hostname;
+		private final Option<Protos.SlaveID> slaveID;
 
-		private TaskState state;
+		private final Option<String> hostname;
 
-		public Worker(Protos.TaskID taskID, Option<Protos.SlaveID> slaveID, Option<String> hostname, TaskState state) {
+		private final WorkerState state;
+
+		private Worker(Protos.TaskID taskID, Option<Protos.SlaveID> slaveID, Option<String> hostname, WorkerState state) {
 			requireNonNull(taskID, "taskID");
 			requireNonNull(slaveID, "slaveID");
 			requireNonNull(hostname, "hostname");
@@ -82,37 +110,61 @@ public interface MesosWorkerStore {
 			this.state = state;
 		}
 
+		/**
+		 * Get the worker's task ID.
+         */
 		public Protos.TaskID taskID() {
 			return taskID;
 		}
 
+		/**
+		 * Get the worker's assigned slave ID.
+         */
 		public Option<Protos.SlaveID> slaveID() {
 			return slaveID;
 		}
 
+		/**
+		 * Get the worker's assigned hostname.
+         */
 		public Option<String> hostname() {
 			return hostname;
 		}
 
-		public TaskState state() {
+		/**
+		 * Get the worker's state.
+         */
+		public WorkerState state() {
 			return state;
 		}
 
 		// valid transition methods
 
-		public static Worker newTask(Protos.TaskID taskID) {
+		/**
+		 * Create a new worker with the given taskID.
+		 * @return a new worker instance.
+		 */
+		public static Worker newWorker(Protos.TaskID taskID) {
 			return new Worker(
 				taskID,
 				Option.<Protos.SlaveID>empty(), Option.<String>empty(),
-				TaskState.New);
+				WorkerState.New);
 		}
 
-		public Worker launchTask(Protos.SlaveID slaveID, String hostname) {
-			return new Worker(taskID, Option.apply(slaveID), Option.apply(hostname), TaskState.Launched);
+		/**
+		 * Transition the worker to a launched state.
+		 * @return a new worker instance (does not mutate the current instance).
+		 */
+		public Worker launchWorker(Protos.SlaveID slaveID, String hostname) {
+			return new Worker(taskID, Option.apply(slaveID), Option.apply(hostname), WorkerState.Launched);
 		}
 
-		public Worker releaseTask() {
-			return new Worker(taskID, slaveID, hostname, TaskState.Released);
+		/**
+		 * Transition the worker to a released state.
+		 * @return a new worker instance (does not mutate the current instance).
+		 */
+		public Worker releaseWorker() {
+			return new Worker(taskID, slaveID, hostname, WorkerState.Released);
 		}
 
 		@Override
@@ -125,14 +177,14 @@ public interface MesosWorkerStore {
 			}
 			Worker worker = (Worker) o;
 			return Objects.equals(taskID, worker.taskID) &&
-				Objects.equals(slaveID.isDefined() ? slaveID.get() : null, worker.slaveID.isDefined() ? worker.slaveID.get() : null) &&
-				Objects.equals(hostname.isDefined() ? hostname.get() : null, worker.hostname.isDefined() ? worker.hostname.get() : null) &&
+				Objects.equals(slaveID, worker.slaveID) &&
+				Objects.equals(hostname, worker.hostname) &&
 				state == worker.state;
 		}
 
 		@Override
 		public int hashCode() {
-			return Objects.hash(taskID, slaveID.isDefined() ? slaveID.get() : null, hostname.isDefined() ? hostname.get() : null, state);
+			return Objects.hash(taskID, slaveID, hostname, state);
 		}
 
 		@Override
@@ -146,7 +198,24 @@ public interface MesosWorkerStore {
 		}
 	}
 
-	enum TaskState {
-		New,Launched,Released
+	/**
+	 * The (planned) state of the worker.
+	 */
+	enum WorkerState {
+
+		/**
+		 * Indicates that the worker is new (not yet launched).
+		 */
+		New,
+
+		/**
+		 * Indicates that the worker is launched.
+		 */
+		Launched,
+
+		/**
+		 * Indicates that the worker is released.
+		 */
+		Released
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/StandaloneMesosWorkerStore.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/StandaloneMesosWorkerStore.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/StandaloneMesosWorkerStore.java
index 9dcc6c0..55e7e28 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/StandaloneMesosWorkerStore.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/StandaloneMesosWorkerStore.java
@@ -46,7 +46,7 @@ public class StandaloneMesosWorkerStore implements MesosWorkerStore {
 	}
 
 	@Override
-	public void stop() throws Exception {
+	public void stop(boolean cleanup) throws Exception {
 
 	}
 
@@ -77,11 +77,8 @@ public class StandaloneMesosWorkerStore implements MesosWorkerStore {
 	}
 
 	@Override
-	public void removeWorker(Protos.TaskID taskID) throws Exception {
-		storedWorkers.remove(taskID);
-	}
-
-	@Override
-	public void cleanup() throws Exception {
+	public boolean removeWorker(Protos.TaskID taskID) throws Exception {
+		Worker prior = storedWorkers.remove(taskID);
+		return prior != null;
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/ZooKeeperMesosWorkerStore.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/ZooKeeperMesosWorkerStore.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/ZooKeeperMesosWorkerStore.java
index 7c689b8..45553d4 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/ZooKeeperMesosWorkerStore.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/store/ZooKeeperMesosWorkerStore.java
@@ -50,7 +50,10 @@ public class ZooKeeperMesosWorkerStore implements MesosWorkerStore {
 
 	private static final Logger LOG = LoggerFactory.getLogger(ZooKeeperMesosWorkerStore.class);
 
-	private final Object cacheLock = new Object();
+	private final Object startStopLock = new Object();
+
+	/** Root store path in ZK. */
+	private final String storePath;
 
 	/** Client (not a namespace facade) */
 	private final CuratorFramework client;
@@ -79,6 +82,7 @@ public class ZooKeeperMesosWorkerStore implements MesosWorkerStore {
 		// Keep a reference to the original client and not the namespace facade. The namespace
 		// facade cannot be closed.
 		this.client = checkNotNull(client, "client");
+		this.storePath = storePath;
 
 		// All operations will have the given path as root
 		client.newNamespaceAwareEnsurePath(storePath).ensure(client.getZookeeperClient());
@@ -94,6 +98,7 @@ public class ZooKeeperMesosWorkerStore implements MesosWorkerStore {
 		facade.newNamespaceAwareEnsurePath("/workers").ensure(client.getZookeeperClient());
 		CuratorFramework storeFacade = client.usingNamespace(facade.getNamespace() + "/workers");
 
+		// using late-binding as a workaround for shaded curator dependency of flink-runtime.
 		this.workersInZooKeeper = ZooKeeperStateHandleStore.class
 			.getConstructor(CuratorFramework.class, StateStorageHelper.class)
 			.newInstance(storeFacade, stateStorage);
@@ -101,7 +106,7 @@ public class ZooKeeperMesosWorkerStore implements MesosWorkerStore {
 
 	@Override
 	public void start() throws Exception {
-		synchronized (cacheLock) {
+		synchronized (startStopLock) {
 			if (!isRunning) {
 				isRunning = true;
 				frameworkIdInZooKeeper.start();
@@ -110,11 +115,17 @@ public class ZooKeeperMesosWorkerStore implements MesosWorkerStore {
 		}
 	}
 
-	public void stop() throws Exception {
-		synchronized (cacheLock) {
+	public void stop(boolean cleanup) throws Exception {
+		synchronized (startStopLock) {
 			if (isRunning) {
 				frameworkIdInZooKeeper.close();
 				totalTaskCountInZooKeeper.close();
+
+				if(cleanup) {
+					workersInZooKeeper.removeAndDiscardAllState();
+					client.delete().deletingChildrenIfNeeded().forPath(storePath);
+				}
+
 				client.close();
 				isRunning = false;
 			}
@@ -132,10 +143,10 @@ public class ZooKeeperMesosWorkerStore implements MesosWorkerStore {
 	 * Get the persisted framework ID.
 	 * @return the current ID or empty if none is yet persisted.
 	 * @throws Exception on ZK failures, interruptions.
-     */
+	 */
 	@Override
 	public Option<Protos.FrameworkID> getFrameworkID() throws Exception {
-		synchronized (cacheLock) {
+		synchronized (startStopLock) {
 			verifyIsRunning();
 
 			Option<Protos.FrameworkID> frameworkID;
@@ -154,10 +165,10 @@ public class ZooKeeperMesosWorkerStore implements MesosWorkerStore {
 	 * Update the persisted framework ID.
 	 * @param frameworkID the new ID or empty to remove the persisted ID.
 	 * @throws Exception on ZK failures, interruptions.
-     */
+	 */
 	@Override
 	public void setFrameworkID(Option<Protos.FrameworkID> frameworkID) throws Exception {
-		synchronized (cacheLock) {
+		synchronized (startStopLock) {
 			verifyIsRunning();
 
 			byte[] value = frameworkID.isDefined() ? frameworkID.get().getValue().getBytes() : new byte[0];
@@ -170,7 +181,7 @@ public class ZooKeeperMesosWorkerStore implements MesosWorkerStore {
 	 */
 	@Override
 	public Protos.TaskID newTaskID() throws Exception {
-		synchronized (cacheLock) {
+		synchronized (startStopLock) {
 			verifyIsRunning();
 
 			int nextCount;
@@ -189,7 +200,7 @@ public class ZooKeeperMesosWorkerStore implements MesosWorkerStore {
 
 	@Override
 	public List<MesosWorkerStore.Worker> recoverWorkers() throws Exception {
-		synchronized (cacheLock) {
+		synchronized (startStopLock) {
 			verifyIsRunning();
 
 			List<Tuple2<StateHandle<MesosWorkerStore.Worker>, String>> handles = workersInZooKeeper.getAll();
@@ -216,7 +227,7 @@ public class ZooKeeperMesosWorkerStore implements MesosWorkerStore {
 		checkNotNull(worker, "worker");
 		String path = getPathForWorker(worker.taskID());
 
-		synchronized (cacheLock) {
+		synchronized (startStopLock) {
 			verifyIsRunning();
 
 			int currentVersion = workersInZooKeeper.exists(path);
@@ -239,22 +250,23 @@ public class ZooKeeperMesosWorkerStore implements MesosWorkerStore {
 	}
 
 	@Override
-	public void removeWorker(Protos.TaskID taskID) throws Exception {
+	public boolean removeWorker(Protos.TaskID taskID) throws Exception {
 		checkNotNull(taskID, "taskID");
 		String path = getPathForWorker(taskID);
-		synchronized (cacheLock) {
+		synchronized (startStopLock) {
 			verifyIsRunning();
 
-			workersInZooKeeper.remove(path);
+			if(workersInZooKeeper.exists(path) == -1) {
+				LOG.debug("No such worker {} in ZooKeeper.", taskID);
+				return false;
+			}
+
+			workersInZooKeeper.removeAndDiscardState(path);
 			LOG.debug("Removed worker {} from ZooKeeper.", taskID);
+			return true;
 		}
 	}
 
-	@Override
-	public void cleanup() throws Exception {
-		// TODO
-	}
-
 	/**
 	 * Get the ZK path for the given task ID (with leading slash).
 	 */
@@ -269,7 +281,7 @@ public class ZooKeeperMesosWorkerStore implements MesosWorkerStore {
 	 * @param configuration the Flink configuration.
 	 * @return a worker store.
 	 * @throws Exception
-     */
+	 */
 	public static ZooKeeperMesosWorkerStore createMesosWorkerStore(
 			CuratorFramework client,
 			Configuration configuration) throws Exception {

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/SchedulerProxy.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/SchedulerProxy.java b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/SchedulerProxy.java
index 3411771..b6d3383 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/SchedulerProxy.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/SchedulerProxy.java
@@ -22,7 +22,8 @@ import akka.actor.ActorRef;
 
 import org.apache.flink.mesos.scheduler.messages.Disconnected;
 import org.apache.flink.mesos.scheduler.messages.Error;
-import org.apache.flink.mesos.scheduler.messages.Error;
+import org.apache.flink.mesos.scheduler.messages.ExecutorLost;
+import org.apache.flink.mesos.scheduler.messages.FrameworkMessage;
 import org.apache.flink.mesos.scheduler.messages.OfferRescinded;
 import org.apache.flink.mesos.scheduler.messages.ReRegistered;
 import org.apache.flink.mesos.scheduler.messages.Registered;
@@ -46,7 +47,7 @@ import java.util.List;
 public class SchedulerProxy implements Scheduler {
 
 	/** The actor to which we report the callbacks */
-	private ActorRef mesosActor;
+	private final ActorRef mesosActor;
 
 	public SchedulerProxy(ActorRef mesosActor) {
 		this.mesosActor = mesosActor;
@@ -67,7 +68,6 @@ public class SchedulerProxy implements Scheduler {
 		mesosActor.tell(new Disconnected(), ActorRef.noSender());
 	}
 
-
 	@Override
 	public void resourceOffers(SchedulerDriver driver, List<Protos.Offer> offers) {
 		mesosActor.tell(new ResourceOffers(offers), ActorRef.noSender());
@@ -85,7 +85,7 @@ public class SchedulerProxy implements Scheduler {
 
 	@Override
 	public void frameworkMessage(SchedulerDriver driver, Protos.ExecutorID executorId, Protos.SlaveID slaveId, byte[] data) {
-		throw new UnsupportedOperationException("frameworkMessage is unexpected");
+		mesosActor.tell(new FrameworkMessage(executorId, slaveId, data), ActorRef.noSender());
 	}
 
 	@Override
@@ -95,7 +95,7 @@ public class SchedulerProxy implements Scheduler {
 
 	@Override
 	public void executorLost(SchedulerDriver driver, Protos.ExecutorID executorId, Protos.SlaveID slaveId, int status) {
-		throw new UnsupportedOperationException("executorLost is unexpected");
+		mesosActor.tell(new ExecutorLost(executorId, slaveId, status), ActorRef.noSender());
 	}
 
 	@Override

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/TaskSchedulerBuilder.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/TaskSchedulerBuilder.java b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/TaskSchedulerBuilder.java
index 80e572d..c841e22 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/TaskSchedulerBuilder.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/TaskSchedulerBuilder.java
@@ -28,7 +28,14 @@ import com.netflix.fenzo.functions.Action1;
  * Note that the Fenzo-provided {@link TaskScheduler.Builder} cannot be mocked, which motivates this interface.
  */
 public interface TaskSchedulerBuilder {
+
+	/**
+	 * Set the callback action for rejecting a lease.
+	 */
 	TaskSchedulerBuilder withLeaseRejectAction(Action1<VirtualMachineLease> action);
 
+	/**
+	 * Build a Fenzo task scheduler.
+	 */
 	TaskScheduler build();
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/AcceptOffers.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/AcceptOffers.java b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/AcceptOffers.java
index 0404f07..dac2803 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/AcceptOffers.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/AcceptOffers.java
@@ -20,17 +20,20 @@ package org.apache.flink.mesos.scheduler.messages;
 
 import org.apache.mesos.Protos;
 
+import java.io.Serializable;
 import java.util.Collection;
 
 /**
  * Local message sent by the launch coordinator to the scheduler to accept offers.
  */
-public class AcceptOffers {
+public class AcceptOffers implements Serializable {
 
-	private String hostname;
-	private Collection<Protos.OfferID> offerIds;
-	private Collection<Protos.Offer.Operation> operations;
-	private Protos.Filters filters;
+	private static final long serialVersionUID = 1L;
+
+	private final String hostname;
+	private final Collection<Protos.OfferID> offerIds;
+	private final Collection<Protos.Offer.Operation> operations;
+	private final Protos.Filters filters;
 
 	public AcceptOffers(String hostname, Collection<Protos.OfferID> offerIds, Collection<Protos.Offer.Operation> operations) {
 		this.hostname = hostname;

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Connected.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Connected.java b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Connected.java
index b51e761..20cd059 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Connected.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Connected.java
@@ -18,9 +18,12 @@
 
 package org.apache.flink.mesos.scheduler.messages;
 
+import java.io.Serializable;
+
 /**
  * Message sent by the callback handler to the scheduler actor
  * when the scheduler becomes "connected" to the master.
  */
-public abstract class Connected {
+public abstract class Connected implements Serializable {
+	private static final long serialVersionUID = 1L;
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Disconnected.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Disconnected.java b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Disconnected.java
index 22664b7..df0f0d9 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Disconnected.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Disconnected.java
@@ -18,11 +18,16 @@
 
 package org.apache.flink.mesos.scheduler.messages;
 
+import java.io.Serializable;
+
 /**
  * Message sent by the callback handler to the scheduler actor
  * when the scheduler becomes "disconnected" from the master (e.g., the master fails and another is taking over).
  */
-public class Disconnected {
+public class Disconnected implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
 	@Override
 	public String toString() {
 		return "Disconnected{}";

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Error.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Error.java b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Error.java
index 83508c1..83896fd 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Error.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Error.java
@@ -18,12 +18,17 @@
 
 package org.apache.flink.mesos.scheduler.messages;
 
+import java.io.Serializable;
+
 /**
  * Message sent when there is an unrecoverable error in the scheduler or
  * driver. The driver will be aborted BEFORE invoking this callback.
  */
-public class Error {
-	private String message;
+public class Error implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	private final String message;
 
 	public Error(String message) {
 		this.message = message;

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/ExecutorLost.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/ExecutorLost.java b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/ExecutorLost.java
new file mode 100644
index 0000000..9cb2e30
--- /dev/null
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/ExecutorLost.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.mesos.scheduler.messages;
+
+import org.apache.mesos.Protos;
+
+import java.io.Serializable;
+
+/**
+ * Message sent when an executor has exited/terminated. Note that any
+ * tasks running will have TASK_LOST status updates automagically
+ * generated.
+ */
+public class ExecutorLost implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	private final Protos.ExecutorID executorId;
+
+	private final Protos.SlaveID slaveId;
+
+	private final int status;
+
+	public ExecutorLost(Protos.ExecutorID executorId, Protos.SlaveID slaveId, int status) {
+		this.executorId = executorId;
+		this.slaveId = slaveId;
+		this.status = status;
+	}
+
+	public Protos.ExecutorID executorId() {
+		return executorId;
+	}
+
+	public Protos.SlaveID slaveId() {
+		return slaveId;
+	}
+
+	public int status() {
+		return status;
+	}
+
+	@Override
+	public String toString() {
+		return "ExecutorLost{" +
+			"executorId=" + executorId +
+			", slaveId=" + slaveId +
+			", status=" + status +
+			'}';
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/FrameworkMessage.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/FrameworkMessage.java b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/FrameworkMessage.java
new file mode 100644
index 0000000..ed74300
--- /dev/null
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/FrameworkMessage.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.mesos.scheduler.messages;
+
+import org.apache.mesos.Protos;
+
+import java.io.Serializable;
+import java.util.Arrays;
+
+/**
+ * Message sent when an executor sends a message. These messages are best
+ * effort; do not expect a framework message to be retransmitted in
+ * any reliable fashion.
+ */
+public class FrameworkMessage implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	private final Protos.ExecutorID executorId;
+
+	private final Protos.SlaveID slaveId;
+
+	private final byte[] data;
+
+	public FrameworkMessage(Protos.ExecutorID executorId, Protos.SlaveID slaveId, byte[] data) {
+		this.executorId = executorId;
+		this.slaveId = slaveId;
+		this.data = data;
+	}
+
+	public Protos.ExecutorID executorId() {
+		return executorId;
+	}
+
+	public Protos.SlaveID slaveId() {
+		return slaveId;
+	}
+
+	public byte[] data() {
+		return data;
+	}
+
+	@Override
+	public String toString() {
+		return "FrameworkMessage{" +
+			"executorId=" + executorId +
+			", slaveId=" + slaveId +
+			", data=" + Arrays.toString(data) +
+			'}';
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/OfferRescinded.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/OfferRescinded.java b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/OfferRescinded.java
index 6ae74c4..1b39b66 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/OfferRescinded.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/OfferRescinded.java
@@ -20,12 +20,17 @@ package org.apache.flink.mesos.scheduler.messages;
 
 import org.apache.mesos.Protos;
 
+import java.io.Serializable;
+
 /**
  * Message sent by the callback handler to the scheduler actor
  * when an offer is no longer valid (e.g., the slave was lost or another framework used resources in the offer).
  */
-public class OfferRescinded {
-	private Protos.OfferID offerId;
+public class OfferRescinded implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	private final Protos.OfferID offerId;
 
 	public OfferRescinded(Protos.OfferID offerId) {
 		this.offerId = offerId;

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/ReRegistered.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/ReRegistered.java b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/ReRegistered.java
index 116fc89..71aa8d6 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/ReRegistered.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/ReRegistered.java
@@ -27,7 +27,10 @@ import static java.util.Objects.requireNonNull;
  * when the scheduler re-registers with a newly elected Mesos master.
  */
 public class ReRegistered extends Connected {
-	private Protos.MasterInfo masterInfo;
+
+	private static final long serialVersionUID = 1L;
+
+	private final Protos.MasterInfo masterInfo;
 
 	public ReRegistered(Protos.MasterInfo masterInfo) {
 		requireNonNull(masterInfo);

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Registered.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Registered.java b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Registered.java
index c2ce214..f38e68a 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Registered.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/Registered.java
@@ -28,8 +28,10 @@ import static java.util.Objects.requireNonNull;
  */
 public class Registered extends Connected {
 
-	private org.apache.mesos.Protos.FrameworkID frameworkId;
-	private Protos.MasterInfo masterInfo;
+	private static final long serialVersionUID = 1L;
+
+	private final Protos.FrameworkID frameworkId;
+	private final Protos.MasterInfo masterInfo;
 
 	public Registered(Protos.FrameworkID frameworkId, Protos.MasterInfo masterInfo) {
 		requireNonNull(frameworkId);

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/ResourceOffers.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/ResourceOffers.java b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/ResourceOffers.java
index f9d21ea..dc5111d 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/ResourceOffers.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/ResourceOffers.java
@@ -20,6 +20,7 @@ package org.apache.flink.mesos.scheduler.messages;
 
 import org.apache.mesos.Protos;
 
+import java.io.Serializable;
 import java.util.List;
 import static java.util.Objects.requireNonNull;
 
@@ -27,8 +28,11 @@ import static java.util.Objects.requireNonNull;
  * Message sent by the callback handler to the scheduler actor
  * when resources have been offered to this framework.
  */
-public class ResourceOffers {
-	private List<Protos.Offer> offers;
+public class ResourceOffers implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	private final List<Protos.Offer> offers;
 
 	public ResourceOffers(List<Protos.Offer> offers) {
 		requireNonNull(offers);

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/SlaveLost.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/SlaveLost.java b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/SlaveLost.java
index 9255e87..a2730c4 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/SlaveLost.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/SlaveLost.java
@@ -20,12 +20,17 @@ package org.apache.flink.mesos.scheduler.messages;
 
 import org.apache.mesos.Protos;
 
+import java.io.Serializable;
+
 /**
  * Message sent by the callback handler to the scheduler actor
  * when a slave has been determined unreachable (e.g., machine failure, network partition).
  */
-public class SlaveLost {
-	private Protos.SlaveID slaveId;
+public class SlaveLost implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	private final Protos.SlaveID slaveId;
 
 	public SlaveLost(Protos.SlaveID slaveId) {
 		this.slaveId = slaveId;

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/StatusUpdate.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/StatusUpdate.java b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/StatusUpdate.java
index 18c9a00..65a2f1b 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/StatusUpdate.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/scheduler/messages/StatusUpdate.java
@@ -20,13 +20,18 @@ package org.apache.flink.mesos.scheduler.messages;
 
 import org.apache.mesos.Protos;
 
+import java.io.Serializable;
+
 /**
  * Message sent by the callback handler to the scheduler actor
  * when the status of a task has changed (e.g., a slave is lost and so the task is lost,
  * a task finishes and an executor sends a status update saying so, etc).
  */
-public class StatusUpdate {
-	private Protos.TaskStatus status;
+public class StatusUpdate implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	private final Protos.TaskStatus status;
 
 	public StatusUpdate(Protos.TaskStatus status) {
 		this.status = status;

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/util/MesosArtifactServer.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/util/MesosArtifactServer.java b/flink-mesos/src/main/java/org/apache/flink/mesos/util/MesosArtifactServer.java
index 63bb7b1..6547cb3 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/util/MesosArtifactServer.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/util/MesosArtifactServer.java
@@ -125,12 +125,12 @@ public class MesosArtifactServer {
 
 		baseURL = new URL("http", serverHostname, port, "/" + sessionID + "/");
 
-		LOG.info("Mesos artifact server listening at " + address + ':' + port);
+		LOG.info("Mesos artifact server listening at {}:{}", address, port);
 	}
 
 	/**
 	 * Get the server port on which the artifact server is listening.
-     */
+	 */
 	public synchronized int getServerPort() {
 		Channel server = this.serverChannel;
 		if (server != null) {
@@ -149,7 +149,7 @@ public class MesosArtifactServer {
 	 * @param remoteFile the remote path with which to locate the file.
 	 * @return the fully-qualified remote path to the file.
 	 * @throws MalformedURLException if the remote path is invalid.
-     */
+	 */
 	public synchronized URL addFile(File localFile, String remoteFile) throws MalformedURLException {
 		URL fileURL = new URL(baseURL, remoteFile);
 		router.ANY(fileURL.getPath(), new VirtualFileServerHandler(localFile));
@@ -159,7 +159,7 @@ public class MesosArtifactServer {
 	/**
 	 * Stops the artifact server.
 	 * @throws Exception
-     */
+	 */
 	public synchronized void stop() throws Exception {
 		if (this.serverChannel != null) {
 			this.serverChannel.close().awaitUninterruptibly();
@@ -179,7 +179,7 @@ public class MesosArtifactServer {
 	@ChannelHandler.Sharable
 	public static class VirtualFileServerHandler extends SimpleChannelInboundHandler<Routed> {
 
-		private File file;
+		private final File file;
 
 		public VirtualFileServerHandler(File file) {
 			this.file = file;
@@ -194,7 +194,7 @@ public class MesosArtifactServer {
 			HttpRequest request = routed.request();
 
 			if (LOG.isDebugEnabled()) {
-				LOG.debug(request.getMethod() + " request for file '" + file.getAbsolutePath() + '\'');
+				LOG.debug("{} request for file '{}'", request.getMethod(), file.getAbsolutePath());
 			}
 
 			if(!(request.getMethod() == GET || request.getMethod() == HEAD)) {

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/util/MesosConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/util/MesosConfiguration.java b/flink-mesos/src/main/java/org/apache/flink/mesos/util/MesosConfiguration.java
index 7961995..40dc41c 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/util/MesosConfiguration.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/util/MesosConfiguration.java
@@ -27,25 +27,27 @@ import scala.Option;
 
 import java.util.Map;
 
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
 /**
  * The typed configuration settings associated with a Mesos scheduler.
  */
 public class MesosConfiguration {
 
-	private String masterUrl;
+	private final String masterUrl;
 
-	private Protos.FrameworkInfo.Builder frameworkInfo;
+	private final Protos.FrameworkInfo.Builder frameworkInfo;
 
-	private Option<Protos.Credential.Builder> credential = Option.empty();
+	private final Option<Protos.Credential.Builder> credential;
 
 	public MesosConfiguration(
 		String masterUrl,
 		Protos.FrameworkInfo.Builder frameworkInfo,
 		Option<Protos.Credential.Builder> credential) {
 
-		this.masterUrl = masterUrl;
-		this.frameworkInfo = frameworkInfo;
-		this.credential = credential;
+		this.masterUrl = checkNotNull(masterUrl);
+		this.frameworkInfo = checkNotNull(frameworkInfo);
+		this.credential = checkNotNull(credential);
 	}
 
 	/**
@@ -89,19 +91,20 @@ public class MesosConfiguration {
 	/**
 	 * Create the Mesos scheduler driver based on this configuration.
 	 * @param scheduler the scheduler to use.
-	 * @param implicitAcknowledgements whether to configure the driver for implicit implicit acknowledgements.
+	 * @param implicitAcknowledgements whether to configure the driver for implicit acknowledgements.
      * @return a scheduler driver.
      */
 	public SchedulerDriver createDriver(Scheduler scheduler, boolean implicitAcknowledgements) {
 		MesosSchedulerDriver schedulerDriver;
 		if(this.credential().isDefined()) {
 			schedulerDriver =
-				new MesosSchedulerDriver(scheduler, frameworkInfo.build(), this.masterUrl(), false,
-					this.credential().get().build());
+				new MesosSchedulerDriver(scheduler, frameworkInfo.build(), this.masterUrl(),
+					implicitAcknowledgements, this.credential().get().build());
 		}
 		else {
 			schedulerDriver =
-				new MesosSchedulerDriver(scheduler, frameworkInfo.build(), this.masterUrl(), false);
+				new MesosSchedulerDriver(scheduler, frameworkInfo.build(), this.masterUrl(),
+					implicitAcknowledgements);
 		}
 		return schedulerDriver;
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/java/org/apache/flink/mesos/util/ZooKeeperUtils.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/util/ZooKeeperUtils.java b/flink-mesos/src/main/java/org/apache/flink/mesos/util/ZooKeeperUtils.java
index ced4a11..6892a65 100644
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/util/ZooKeeperUtils.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/util/ZooKeeperUtils.java
@@ -33,7 +33,7 @@ public class ZooKeeperUtils {
 	@SuppressWarnings("unchecked")
 	public static CuratorFramework startCuratorFramework(Configuration configuration) {
 
-		// workaround for shaded curator dependency of flink-runtime
+		// using late-binding as a workaround for shaded curator dependency of flink-runtime
 		Object client = org.apache.flink.runtime.util.ZooKeeperUtils.startCuratorFramework(configuration);
 		return (CuratorFramework) client;
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManager.scala b/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManager.scala
index 3e1054c..d8b6775 100644
--- a/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManager.scala
+++ b/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManager.scala
@@ -30,14 +30,14 @@ import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerConfigurati
   * messages.
   */
 class MesosTaskManager(
-                       config: TaskManagerConfiguration,
-                       resourceID: ResourceID,
-                       connectionInfo: InstanceConnectionInfo,
-                       memoryManager: MemoryManager,
-                       ioManager: IOManager,
-                       network: NetworkEnvironment,
-                       numberOfSlots: Int,
-                       leaderRetrievalService: LeaderRetrievalService)
+    config: TaskManagerConfiguration,
+    resourceID: ResourceID,
+    connectionInfo: InstanceConnectionInfo,
+    memoryManager: MemoryManager,
+    ioManager: IOManager,
+    network: NetworkEnvironment,
+    numberOfSlots: Int,
+    leaderRetrievalService: LeaderRetrievalService)
   extends TaskManager(
     config,
     resourceID,

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/LaunchCoordinator.scala
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/LaunchCoordinator.scala b/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/LaunchCoordinator.scala
index bee25d3..1024b5c 100644
--- a/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/LaunchCoordinator.scala
+++ b/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/LaunchCoordinator.scala
@@ -18,6 +18,8 @@
 
 package org.apache.flink.mesos.scheduler
 
+import java.util.Collections
+
 import akka.actor.{Actor, ActorRef, FSM, Props}
 import com.netflix.fenzo._
 import com.netflix.fenzo.functions.Action1
@@ -27,7 +29,6 @@ import org.apache.flink.api.java.tuple.{Tuple2=>FlinkTuple2}
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.mesos.scheduler.LaunchCoordinator._
 import org.apache.flink.mesos.scheduler.messages._
-import org.apache.mesos.Protos.TaskInfo
 import org.apache.mesos.{SchedulerDriver, Protos}
 
 import scala.collection.JavaConverters._
@@ -38,9 +39,10 @@ import scala.concurrent.duration._
   * The launch coordinator handles offer processing, including
   * matching offers to tasks and making reservations.
   *
-  * The coordinator uses Netflix Fenzo to optimize task placement.   During the GatheringOffers phase,
-  * offers are evaluated by Fenzo for suitability to the planned tasks.   Reservations are then placed
-  * against the best offers, leading to revised offers containing reserved resources with which to launch task(s).
+  * The coordinator uses Netflix Fenzo to optimize task placement.   During the GatheringOffers
+  * phase, offers are evaluated by Fenzo for suitability to the planned tasks.   Reservations are
+  * then placed against the best offers, leading to revised offers containing reserved resources
+  * with which to launch task(s).
   */
 class LaunchCoordinator(
     manager: ActorRef,
@@ -62,7 +64,8 @@ class LaunchCoordinator(
     optimizerBuilder
       .withLeaseRejectAction(new Action1[VirtualMachineLease]() {
         def call(lease: VirtualMachineLease) {
-          LOG.info(s"Declined offer ${lease.getId} from ${lease.hostname()} of ${lease.memoryMB()} MB, ${lease.cpuCores()} cpus.")
+          LOG.info(s"Declined offer ${lease.getId} from ${lease.hostname()} "
+            + s"of ${lease.memoryMB()} MB, ${lease.cpuCores()} cpus.")
           schedulerDriver.declineOffer(lease.getOffer.getId)
         }
       }).build
@@ -81,7 +84,7 @@ class LaunchCoordinator(
   /**
     * State: Suspended
     *
-    * Wait for (re-)connection to Mesos.   No offers exist in this state, but outstanding tasks might.
+    * Wait for (re-)connection to Mesos. No offers exist in this state, but outstanding tasks might.
     */
   when(Suspended) {
     case Event(msg: Connected, data: GatherData) =>
@@ -127,6 +130,7 @@ class LaunchCoordinator(
       assert(nextStateData.newLeases.isEmpty)
       schedulerDriver.suppressOffers()
       optimizer.expireAllLeases()
+      optimizer.scheduleOnce(Collections.emptyList(), Collections.emptyList())
   }
 
   /**
@@ -143,20 +147,24 @@ class LaunchCoordinator(
       goto(Suspended) using data.copy(newLeases = Nil)
 
     case Event(offers: ResourceOffers, data: GatherData) =>
-      val leases = offers.offers().asScala.map(new VMLeaseObject(_).asInstanceOf[VirtualMachineLease])
+      val leases = offers.offers().asScala.map(
+        new VMLeaseObject(_).asInstanceOf[VirtualMachineLease])
       if(LOG.isInfoEnabled) {
-        val (cpus, mem) = leases.foldLeft((0.0,0.0)) { (z,o) => (z._1 + o.cpuCores(), z._2 + o.memoryMB()) }
+        val (cpus, mem) = leases.foldLeft((0.0,0.0)) {
+          (z,o) => (z._1 + o.cpuCores(), z._2 + o.memoryMB())
+        }
         LOG.info(s"Received offer(s) of $mem MB, $cpus cpus:")
-        for(lease <- leases) {
-          LOG.info(s"  ${lease.getId} from ${lease.hostname()} of ${lease.memoryMB()} MB, ${lease.cpuCores()} cpus")
+        for(l <- leases) {
+          LOG.info(s"  ${l.getId} from ${l.hostname()} of ${l.memoryMB()} MB, ${l.cpuCores()} cpus")
         }
       }
-      stay using data.copy(newLeases = data.newLeases ++ leases)
+      stay using data.copy(newLeases = data.newLeases ++ leases) forMax (1 seconds)
 
     case Event(StateTimeout, data: GatherData) =>
       val remaining = MutableMap(data.tasks.map(t => t.taskRequest.getId -> t):_*)
 
-      LOG.info(s"Processing ${remaining.size} task(s) against ${data.newLeases.length} new offer(s) plus outstanding offers.")
+      LOG.info(s"Processing ${remaining.size} task(s) against ${data.newLeases.length}"
+        + s" new offer(s) plus outstanding offers.")
 
       // attempt to assign the outstanding tasks using the optimizer
       val result = optimizer.scheduleOnce(
@@ -193,7 +201,8 @@ class LaunchCoordinator(
         manager ! new AcceptOffers(hostname, offerIds.asJava, operations.asJava)
 
         if(LOG.isInfoEnabled) {
-          LOG.info(s"Launched ${launchedTasks.length} task(s) on ${hostname} using ${offerIds.length} offer(s):")
+          LOG.info(s"Launched ${launchedTasks.length} task(s) on ${hostname}"
+            + s" using ${offerIds.length} offer(s):")
           for(offerId <- offerIds) {
             LOG.info(s"  ${offerId.getValue}")
           }
@@ -206,7 +215,7 @@ class LaunchCoordinator(
       } else {
         LOG.info(s"Waiting for more offers; ${remaining.size} task(s) are not yet launched.")
 
-        stay using data.copy(newLeases = Nil, tasks = remaining.values.toList) forMax SUBSEQUENT_GATHER_DURATION
+        stay using data.copy(newLeases = Nil, tasks = remaining.values.toList)
       }
   }
 
@@ -222,7 +231,8 @@ class LaunchCoordinator(
       // forget rescinded offers
       LOG.info(s"Offer ${offer.offerId()} was rescinded.")
       optimizer.expireLease(offer.offerId().getValue)
-      stay using data.copy(newLeases = data.newLeases.filterNot(_.getOffer.getId == offer.offerId()))
+      stay using data.copy(
+        newLeases = data.newLeases.filterNot(_.getOffer.getId == offer.offerId()))
 
     case Event(msg: Assign, _) =>
       // recovering an earlier task assignment
@@ -250,7 +260,6 @@ class LaunchCoordinator(
 object LaunchCoordinator {
 
   val GATHER_DURATION = 5.seconds
-  val SUBSEQUENT_GATHER_DURATION = 5.seconds
 
   // ------------------------------------------------------------------------
   //  FSM State
@@ -284,15 +293,17 @@ object LaunchCoordinator {
   }
 
   /**
-    * Informs the launch coordinator that some task(s) are assigned to a host (for planning purposes).
+    * Informs the launch coordinator that some task(s) are assigned
+    * to a host (for planning purposes).
     *
-    * This is sent by the RM in recovery procedures to recover the optimizer state.  In normal operation,
-    * the launch coordinator itself updates the optimizer state.
+    * This is sent by the RM in recovery procedures to recover the optimizer state.
+    * In normal operation, the launch coordinator itself updates the optimizer state.
     */
   case class Assign(tasks: java.util.List[FlinkTuple2[TaskRequest, String]])
 
   /**
-    * Informs the launch coordinator that some task is no longer assigned to a host (for planning purposes).
+    * Informs the launch coordinator that some task is no longer assigned
+    * to a host (for planning purposes).
     */
   case class Unassign(taskID: Protos.TaskID, hostname: String)
 
@@ -319,11 +330,13 @@ object LaunchCoordinator {
       allTasks(assignment.getTaskId).launch(slaveId, assignment)
     }
 
-    val launches = Protos.Offer.Operation.newBuilder().setType(Protos.Offer.Operation.Type.LAUNCH).setLaunch(
-      Protos.Offer.Operation.Launch.newBuilder().addAllTaskInfos(
-        assignments.getTasksAssigned.asScala.map(taskInfo).asJava
-      )
-    ).build()
+    val launches = Protos.Offer.Operation.newBuilder()
+      .setType(Protos.Offer.Operation.Type.LAUNCH)
+      .setLaunch(
+        Protos.Offer.Operation.Launch.newBuilder().addAllTaskInfos(
+          assignments.getTasksAssigned.asScala.map(taskInfo).asJava
+        ))
+      .build()
 
     Seq(launches)
   }

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/ReconciliationCoordinator.scala
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/ReconciliationCoordinator.scala b/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/ReconciliationCoordinator.scala
index 239e040..2ce7225 100644
--- a/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/ReconciliationCoordinator.scala
+++ b/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/ReconciliationCoordinator.scala
@@ -48,7 +48,8 @@ class ReconciliationCoordinator(
   when(Suspended) {
     case Event(reconcile: Reconcile, data: ReconciliationData) =>
       val tasks = reconcile.tasks.map(task => (task.getTaskId,task))
-      stay using data.copy(remaining = if(reconcile.replace) tasks.toMap else data.remaining ++ tasks)
+      stay using data.copy(
+        remaining = if(reconcile.replace) tasks.toMap else data.remaining ++ tasks)
 
     case Event(msg: Connected, data: ReconciliationData) =>
       if(data.remaining.nonEmpty) goto(Reconciling)
@@ -75,7 +76,8 @@ class ReconciliationCoordinator(
       // initiate reconciliation for additional tasks (even while reconciliation is ongoing)
       schedulerDriver.reconcileTasks(reconcile.tasks.asJavaCollection)
       val tasks = reconcile.tasks.map(task => (task.getTaskId,task))
-      stay using data.copy(remaining = if(reconcile.replace) tasks.toMap else data.remaining ++ tasks)
+      stay using data.copy(
+        remaining = if(reconcile.replace) tasks.toMap else data.remaining ++ tasks)
 
     case Event(update: StatusUpdate, data: ReconciliationData) =>
       // status information arrived for a task
@@ -143,7 +145,9 @@ object ReconciliationCoordinator {
     * @param remaining
     * @param retries
     */
-  case class ReconciliationData(remaining: Map[Protos.TaskID,Protos.TaskStatus] = Map(), retries: Int = 0)
+  case class ReconciliationData(
+      remaining: Map[Protos.TaskID,Protos.TaskStatus] = Map(),
+      retries: Int = 0)
 
   /**
     * Initiates the task reconciliation process.

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/TaskMonitor.scala
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/TaskMonitor.scala b/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/TaskMonitor.scala
index ec79585..7840fd4 100644
--- a/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/TaskMonitor.scala
+++ b/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/TaskMonitor.scala
@@ -34,14 +34,15 @@ import scala.concurrent.duration._
 /**
   * Monitors a Mesos task throughout its lifecycle.
   *
-  * Models a task with a state machine reflecting the perceived state of the task in Mesos.   The state
-  * is primarily updated when task status information arrives from Mesos.
+  * Models a task with a state machine reflecting the perceived state of the task in Mesos.
+  * The state is primarily updated when task status information arrives from Mesos.
   *
-  * The associated state data primarily tracks the task's goal (intended) state, as persisted by the scheduler.
-  * Keep in mind that goal state is persisted before actions are taken.    The goal state strictly transitions
-  * thru New->Launched->Released.
+  * The associated state data primarily tracks the task's goal (intended) state, as
+  * persisted by the scheduler. Keep in mind that goal state is persisted before actions are taken.
+  * The goal state strictly transitions thru New->Launched->Released.
   *
-  * Unlike most exchanges with Mesos, task status is delivered at-least-once, so status handling should be idempotent.
+  * Unlike most exchanges with Mesos, task status is delivered at-least-once,
+  * so status handling should be idempotent.
   */
 class TaskMonitor(
     flinkConfig: Configuration,
@@ -103,7 +104,8 @@ class TaskMonitor(
 
   when(Staging, stateTimeout = LAUNCH_TIMEOUT) {
     case Event(StateTimeout, _) =>
-      LOG.warn(s"Mesos task ${stateData.goal.taskID.getValue} didn't launch as expected; reconciling.")
+      LOG.warn(s"Mesos task ${stateData.goal.taskID.getValue} didn't launch as expected;"
+        + s" reconciling.")
 
       // likely cause: the task launch message was dropped - docs suggest reconciliation
       goto(Reconciling)

http://git-wip-us.apache.org/repos/asf/flink/blob/59eeea4c/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/Tasks.scala
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/Tasks.scala b/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/Tasks.scala
index 14147a5..4f49c16 100644
--- a/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/Tasks.scala
+++ b/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/Tasks.scala
@@ -18,11 +18,11 @@
 
 package org.apache.flink.mesos.scheduler
 
-import akka.actor.{Actor, ActorRef, Props}
+import akka.actor.SupervisorStrategy.Escalate
+import akka.actor._
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.mesos.scheduler.ReconciliationCoordinator.Reconcile
-import org.apache.flink.mesos.scheduler.TaskMonitor.{TaskGoalState, TaskGoalStateUpdated, TaskTerminated}
-import org.apache.flink.mesos.scheduler.Tasks._
+import org.apache.flink.mesos.scheduler.TaskMonitor._
 import org.apache.flink.mesos.scheduler.messages._
 import org.apache.mesos.{SchedulerDriver, Protos}
 
@@ -33,23 +33,23 @@ import scala.collection.mutable.{Map => MutableMap}
   *
   * Routes messages between the scheduler and individual task monitor actors.
   */
-class Tasks[M <: TaskMonitor](
+class Tasks(
      flinkConfig: Configuration,
      schedulerDriver: SchedulerDriver,
-     taskMonitorClass: Class[M]) extends Actor {
+     taskMonitorCreator: (ActorRefFactory,TaskGoalState) => ActorRef) extends Actor {
 
   /**
     * A map of task monitors by task ID.
     */
-  private val taskMap: MutableMap[Protos.TaskID,ActorRef] = MutableMap()
+  private[scheduler] val taskMap: MutableMap[Protos.TaskID,ActorRef] = MutableMap()
 
   /**
     * Cache of current connection state.
     */
-  private var registered: Option[Any] = None
+  private[scheduler] var registered: Option[Any] = None
 
-  override def preStart(): Unit = {
-    // TODO subscribe to context.system.deadLetters for messages to nonexistent tasks
+  override def supervisorStrategy: SupervisorStrategy = AllForOneStrategy() {
+    case _ => Escalate
   }
 
   override def receive: Receive = {
@@ -65,27 +65,42 @@ class Tasks[M <: TaskMonitor](
     case msg: TaskGoalStateUpdated =>
       val taskID = msg.state.taskID
 
-      // ensure task monitor exists
-      if(!taskMap.contains(taskID)) {
-        val actorRef = createTask(msg.state)
-        registered.foreach(actorRef ! _)
-      }
+      taskMap.get(taskID) match {
+        case None =>
+          // create a new actor to monitor the task, with the appropriate initial goal state
+          val actorRef = createTask(msg.state)
+          registered.foreach(actorRef ! _)
 
-      taskMap(taskID) ! msg
+        case Some(actor) =>
+          // tell the actor to update its goal state
+          actor ! msg
+      }
 
     case msg: StatusUpdate =>
-      taskMap(msg.status().getTaskId) ! msg
+      taskMap.get(msg.status().getTaskId) match {
+        case Some(ref) =>
+          // tell the actor about the status change
+          ref ! msg
+        case None =>
+          // a status update was received for an unrecognized task, which may occur
+          // when a task is resurrected (i.e. the Mesos master isn't using a strict registry).
+          // see the Mesos reconciliation guide for more information.
+
+          // create a monitor to reliably terminate the resurrected task
+          val actorRef = createTask(Released(msg.status().getTaskId, msg.status().getSlaveId))
+          registered.foreach(actorRef ! _)
+      }
 
     case msg: Reconcile =>
       context.parent.forward(msg)
 
     case msg: TaskTerminated =>
+      taskMap.remove(msg.taskID)
       context.parent.forward(msg)
   }
 
   private def createTask(task: TaskGoalState): ActorRef = {
-    val actorProps = TaskMonitor.createActorProps(taskMonitorClass, flinkConfig, schedulerDriver, task)
-    val actorRef = context.actorOf(actorProps, name = actorName(task.taskID))
+    val actorRef = taskMonitorCreator(context, task)
     taskMap.put(task.taskID, actorRef)
     actorRef
   }
@@ -94,21 +109,19 @@ class Tasks[M <: TaskMonitor](
 object Tasks {
 
   /**
-    * Extract the actor name for the given task ID.
-    */
-  def actorName(taskID: Protos.TaskID): String = {
-    taskID.getValue
-  }
-
-  /**
     * Create a tasks actor.
     */
-  def createActorProps[T <: Tasks[M], M <: TaskMonitor](
+  def createActorProps[T <: Tasks, M <: TaskMonitor](
       actorClass: Class[T],
       flinkConfig: Configuration,
       schedulerDriver: SchedulerDriver,
       taskMonitorClass: Class[M]): Props = {
 
-    Props.create(actorClass, flinkConfig, schedulerDriver, taskMonitorClass)
+    val taskMonitorCreator = (factory: ActorRefFactory, task: TaskGoalState) => {
+      val props = TaskMonitor.createActorProps(taskMonitorClass, flinkConfig, schedulerDriver, task)
+      factory.actorOf(props)
+    }
+
+    Props.create(actorClass, flinkConfig, schedulerDriver, taskMonitorCreator)
   }
 }