You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2018/02/15 10:42:44 UTC

[01/22] flink git commit: [FLINK-8603] [flip6] Split submitJob into job submission and execution result retrieval

Repository: flink
Updated Branches:
  refs/heads/master c7d2a0da4 -> 341467dd5


[FLINK-8603] [flip6] Split submitJob into job submission and execution result retrieval

Split RestClusterClient#submitJob into submitJob and requestJobResult which can
be called individually.

This closes #5428.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b3122f26
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b3122f26
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b3122f26

Branch: refs/heads/master
Commit: b3122f26a8519169ed61e0ce95235a59c677c9bc
Parents: 42f49ac
Author: Till Rohrmann <tr...@apache.org>
Authored: Tue Feb 6 16:46:52 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Thu Feb 15 08:44:17 2018 +0100

----------------------------------------------------------------------
 .../client/program/rest/RestClusterClient.java  | 133 ++++++++++++-------
 1 file changed, 83 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/b3122f26/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
index 564990f..18cd5c0 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
@@ -96,6 +96,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -198,31 +199,22 @@ public class RestClusterClient<T> extends ClusterClient<T> {
 	protected JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException {
 		log.info("Submitting job.");
 		try {
-			// we have to enable queued scheduling because slot will be allocated lazily
-			jobGraph.setAllowQueuedScheduling(true);
-			submitJob(jobGraph);
-		} catch (JobSubmissionException e) {
-			throw new ProgramInvocationException(e);
+			submitJob(jobGraph).get();
+		} catch (InterruptedException | ExecutionException e) {
+			throw new ProgramInvocationException(ExceptionUtils.stripExecutionException(e));
 		}
 
-		final JobResult jobExecutionResult;
+		final CompletableFuture<JobResult> jobResultFuture = requestJobResult(jobGraph.getJobID());
+
+		final JobResult jobResult;
 		try {
-			jobExecutionResult = pollResourceAsync(
-				() -> {
-					final JobMessageParameters messageParameters = new JobMessageParameters();
-					messageParameters.jobPathParameter.resolve(jobGraph.getJobID());
-					return sendRetryableRequest(
-						JobExecutionResultHeaders.getInstance(),
-						messageParameters,
-						EmptyRequestBody.getInstance(),
-						isConnectionProblemException().or(isHttpStatusUnsuccessfulException()));
-				}).get();
-		} catch (final Exception e) {
+			jobResult = jobResultFuture.get();
+		} catch (Exception e) {
 			throw new ProgramInvocationException(e);
 		}
 
-		if (jobExecutionResult.getSerializedThrowable().isPresent()) {
-			final SerializedThrowable serializedThrowable = jobExecutionResult.getSerializedThrowable().get();
+		if (jobResult.getSerializedThrowable().isPresent()) {
+			final SerializedThrowable serializedThrowable = jobResult.getSerializedThrowable().get();
 			final Throwable throwable = serializedThrowable.deserializeError(classLoader);
 			throw new ProgramInvocationException(throwable);
 		}
@@ -231,10 +223,10 @@ public class RestClusterClient<T> extends ClusterClient<T> {
 			// don't return just a JobSubmissionResult here, the signature is lying
 			// The CliFrontend expects this to be a JobExecutionResult
 			this.lastJobExecutionResult = new JobExecutionResult(
-				jobExecutionResult.getJobId(),
-				jobExecutionResult.getNetRuntime(),
+				jobResult.getJobId(),
+				jobResult.getNetRuntime(),
 				AccumulatorHelper.deserializeAccumulators(
-					jobExecutionResult.getAccumulatorResults(),
+					jobResult.getAccumulatorResults(),
 					classLoader));
 			return lastJobExecutionResult;
 		} catch (IOException | ClassNotFoundException e) {
@@ -242,37 +234,78 @@ public class RestClusterClient<T> extends ClusterClient<T> {
 		}
 	}
 
-	private void submitJob(JobGraph jobGraph) throws JobSubmissionException {
+	/**
+	 * Requests the {@link JobResult} for the given {@link JobID}. The method retries multiple
+	 * times to poll the {@link JobResult} before giving up.
+	 *
+	 * @param jobId specifying the job for which to retrieve the {@link JobResult}
+	 * @return Future which is completed with the {@link JobResult} once the job has completed or
+	 * with a failure if the {@link JobResult} could not be retrieved.
+	 */
+	public CompletableFuture<JobResult> requestJobResult(JobID jobId) {
+		return pollResourceAsync(
+			() -> {
+				final JobMessageParameters messageParameters = new JobMessageParameters();
+				messageParameters.jobPathParameter.resolve(jobId);
+				return sendRetryableRequest(
+					JobExecutionResultHeaders.getInstance(),
+					messageParameters,
+					EmptyRequestBody.getInstance(),
+					isConnectionProblemException().or(isHttpStatusUnsuccessfulException()));
+			});
+	}
+
+	/**
+	 * Submits the given {@link JobGraph} to the dispatcher.
+	 *
+	 * @param jobGraph to submit
+	 * @return Future which is completed with the submission response
+	 */
+	public CompletableFuture<JobSubmitResponseBody> submitJob(JobGraph jobGraph) {
+		// we have to enable queued scheduling because slot will be allocated lazily
+		jobGraph.setAllowQueuedScheduling(true);
+
 		log.info("Requesting blob server port.");
-		int blobServerPort;
-		try {
-			CompletableFuture<BlobServerPortResponseBody> portFuture = sendRequest(
-				BlobServerPortHeaders.getInstance());
-			blobServerPort = portFuture.get(timeout.toMillis(), TimeUnit.MILLISECONDS).port;
-		} catch (Exception e) {
-			throw new JobSubmissionException(jobGraph.getJobID(), "Failed to retrieve blob server port.", e);
-		}
+		CompletableFuture<BlobServerPortResponseBody> portFuture = sendRequest(
+			BlobServerPortHeaders.getInstance());
+
+		CompletableFuture<JobGraph> jobUploadFuture = portFuture.thenCombine(
+			getDispatcherAddress(),
+			(BlobServerPortResponseBody response, String dispatcherAddress) -> {
+				log.info("Uploading jar files.");
+				final int blobServerPort = response.port;
+				final InetSocketAddress address = new InetSocketAddress(dispatcherAddress, blobServerPort);
+				final List<PermanentBlobKey> keys;
+				try {
+					keys = BlobClient.uploadJarFiles(address, flinkConfig, jobGraph.getJobID(), jobGraph.getUserJars());
+				} catch (IOException ioe) {
+					throw new CompletionException(new FlinkException("Could not upload job jar files.", ioe));
+				}
 
-		log.info("Uploading jar files.");
-		try {
-			InetSocketAddress address = new InetSocketAddress(getDispatcherAddress().get(), blobServerPort);
-			List<PermanentBlobKey> keys = BlobClient.uploadJarFiles(address, this.flinkConfig, jobGraph.getJobID(), jobGraph.getUserJars());
-			for (PermanentBlobKey key : keys) {
-				jobGraph.addBlob(key);
-			}
-		} catch (Exception e) {
-			throw new JobSubmissionException(jobGraph.getJobID(), "Failed to upload user jars to blob server.", e);
-		}
+				for (PermanentBlobKey key : keys) {
+					jobGraph.addBlob(key);
+				}
 
-		log.info("Submitting job graph.");
-		try {
-			CompletableFuture<JobSubmitResponseBody> responseFuture = sendRequest(
-				JobSubmitHeaders.getInstance(),
-				new JobSubmitRequestBody(jobGraph));
-			responseFuture.get(timeout.toMillis(), TimeUnit.MILLISECONDS);
-		} catch (Exception e) {
-			throw new JobSubmissionException(jobGraph.getJobID(), "Failed to submit JobGraph.", e);
-		}
+				return jobGraph;
+			});
+
+		CompletableFuture<JobSubmitResponseBody> submissionFuture = jobUploadFuture.thenCompose(
+			(JobGraph jobGraphToSubmit) -> {
+				log.info("Submitting job graph.");
+
+				try {
+					return sendRequest(
+						JobSubmitHeaders.getInstance(),
+						new JobSubmitRequestBody(jobGraph));
+				} catch (IOException ioe) {
+					throw new CompletionException(new FlinkException("Could not create JobSubmitRequestBody.", ioe));
+				}
+			});
+
+		return submissionFuture.exceptionally(
+			(Throwable throwable) -> {
+				throw new CompletionException(new JobSubmissionException(jobGraph.getJobID(), "Failed to submit JobGraph.", throwable));
+			});
 	}
 
 	@Override


[08/22] flink git commit: [FLINK-8604] [rest] Move JobTerminationHandler into WebMonitorEndpoint

Posted by tr...@apache.org.
[FLINK-8604] [rest] Move JobTerminationHandler into WebMonitorEndpoint

Register the JobTerminationHandler at the WebMonitorEndpoint to make it accessible to
all REST endpoints.

This closes #5429.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a5105894
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a5105894
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a5105894

Branch: refs/heads/master
Commit: a5105894b981e67631145d1921cd8cbf79fb2308
Parents: eeb453a
Author: Till Rohrmann <tr...@apache.org>
Authored: Thu Feb 8 13:43:09 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Thu Feb 15 08:44:18 2018 +0100

----------------------------------------------------------------------
 .../runtime/dispatcher/DispatcherGateway.java   | 18 ------
 .../dispatcher/DispatcherRestEndpoint.java      | 10 ----
 .../flink/runtime/jobmaster/JobMaster.java      | 18 ++++++
 .../rest/handler/job/JobTerminationHandler.java |  8 +--
 .../runtime/webmonitor/RestfulGateway.java      | 19 +++++++
 .../runtime/webmonitor/WebMonitorEndpoint.java  | 10 ++++
 .../job/JobVertexBackPressureHandlerTest.java   |  2 +-
 .../webmonitor/TestingRestfulGateway.java       | 59 ++++++++++++++++++--
 8 files changed, 106 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/a5105894/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherGateway.java
index 9676429..5f5ad1c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherGateway.java
@@ -57,24 +57,6 @@ public interface DispatcherGateway extends FencedRpcGateway<DispatcherId>, Restf
 		@RpcTimeout Time timeout);
 
 	/**
-	 * Cancel the given job.
-	 *
-	 * @param jobId identifying the job to cancel
-	 * @param timeout of the operation
-	 * @return A future acknowledge if the cancellation succeeded
-	 */
-	CompletableFuture<Acknowledge> cancelJob(JobID jobId, @RpcTimeout Time timeout);
-
-	/**
-	 * Stop the given job.
-	 *
-	 * @param jobId identifying the job to stop
-	 * @param timeout of the operation
-	 * @return A future acknowledge if the stopping succeeded
-	 */
-	CompletableFuture<Acknowledge> stopJob(JobID jobId, @RpcTimeout Time timeout);
-
-	/**
 	 * Returns the port of the blob server.
 	 *
 	 * @param timeout of the operation

http://git-wip-us.apache.org/repos/asf/flink/blob/a5105894/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java
index f9263c9..90022ed 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java
@@ -29,8 +29,6 @@ import org.apache.flink.runtime.rest.handler.RestHandlerConfiguration;
 import org.apache.flink.runtime.rest.handler.RestHandlerSpecification;
 import org.apache.flink.runtime.rest.handler.job.BlobServerPortHandler;
 import org.apache.flink.runtime.rest.handler.job.JobSubmitHandler;
-import org.apache.flink.runtime.rest.handler.job.JobTerminationHandler;
-import org.apache.flink.runtime.rest.messages.JobTerminationHeaders;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.webmonitor.WebMonitorEndpoint;
 import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
@@ -81,13 +79,6 @@ public class DispatcherRestEndpoint extends WebMonitorEndpoint<DispatcherGateway
 		final Time timeout = restConfiguration.getTimeout();
 		final Map<String, String> responseHeaders = restConfiguration.getResponseHeaders();
 
-		JobTerminationHandler jobTerminationHandler = new JobTerminationHandler(
-			restAddressFuture,
-			leaderRetriever,
-			timeout,
-			responseHeaders,
-			JobTerminationHeaders.getInstance());
-
 		BlobServerPortHandler blobServerPortHandler = new BlobServerPortHandler(
 			restAddressFuture,
 			leaderRetriever,
@@ -100,7 +91,6 @@ public class DispatcherRestEndpoint extends WebMonitorEndpoint<DispatcherGateway
 			timeout,
 			responseHeaders);
 
-		handlers.add(Tuple2.of(JobTerminationHeaders.getInstance(), jobTerminationHandler));
 		handlers.add(Tuple2.of(blobServerPortHandler.getMessageHeaders(), blobServerPortHandler));
 		handlers.add(Tuple2.of(jobSubmitHandler.getMessageHeaders(), jobSubmitHandler));
 

http://git-wip-us.apache.org/repos/asf/flink/blob/a5105894/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index bf24dc6..e7583d0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -823,6 +823,24 @@ public class JobMaster extends FencedRpcEndpoint<JobMasterId> implements JobMast
 	//----------------------------------------------------------------------------------------------
 
 	@Override
+	public CompletableFuture<Acknowledge> cancelJob(JobID jobId, Time timeout) {
+		if (jobGraph.getJobID().equals(jobId)) {
+			return cancel(timeout);
+		} else {
+			return FutureUtils.completedExceptionally(new FlinkJobNotFoundException(jobId));
+		}
+	}
+
+	@Override
+	public CompletableFuture<Acknowledge> stopJob(JobID jobId, Time timeout) {
+		if (jobGraph.getJobID().equals(jobId)) {
+			return stop(timeout);
+		} else {
+			return FutureUtils.completedExceptionally(new FlinkJobNotFoundException(jobId));
+		}
+	}
+
+	@Override
 	public CompletableFuture<String> requestRestAddress(Time timeout) {
 		return restAddressFuture;
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/a5105894/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java
index 0998177..96f57f8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java
@@ -21,7 +21,6 @@ package org.apache.flink.runtime.rest.handler.job;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.runtime.concurrent.FutureUtils;
-import org.apache.flink.runtime.dispatcher.DispatcherGateway;
 import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.messages.FlinkJobNotFoundException;
 import org.apache.flink.runtime.rest.handler.AbstractRestHandler;
@@ -33,6 +32,7 @@ import org.apache.flink.runtime.rest.messages.JobIDPathParameter;
 import org.apache.flink.runtime.rest.messages.JobTerminationMessageParameters;
 import org.apache.flink.runtime.rest.messages.MessageHeaders;
 import org.apache.flink.runtime.rest.messages.TerminationModeQueryParameter;
+import org.apache.flink.runtime.webmonitor.RestfulGateway;
 import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
 import org.apache.flink.util.ExceptionUtils;
 
@@ -47,11 +47,11 @@ import java.util.concurrent.TimeoutException;
 /**
  * Request handler for the cancel and stop request.
  */
-public class JobTerminationHandler extends AbstractRestHandler<DispatcherGateway, EmptyRequestBody, EmptyResponseBody, JobTerminationMessageParameters> {
+public class JobTerminationHandler extends AbstractRestHandler<RestfulGateway, EmptyRequestBody, EmptyResponseBody, JobTerminationMessageParameters> {
 
 	public JobTerminationHandler(
 			CompletableFuture<String> localRestAddress,
-			GatewayRetriever<DispatcherGateway> leaderRetriever,
+			GatewayRetriever<? extends RestfulGateway> leaderRetriever,
 			Time timeout,
 			Map<String, String> headers,
 			MessageHeaders<EmptyRequestBody, EmptyResponseBody, JobTerminationMessageParameters> messageHeaders) {
@@ -59,7 +59,7 @@ public class JobTerminationHandler extends AbstractRestHandler<DispatcherGateway
 	}
 
 	@Override
-	public CompletableFuture<EmptyResponseBody> handleRequest(HandlerRequest<EmptyRequestBody, JobTerminationMessageParameters> request, DispatcherGateway gateway) {
+	public CompletableFuture<EmptyResponseBody> handleRequest(HandlerRequest<EmptyRequestBody, JobTerminationMessageParameters> request, RestfulGateway gateway) {
 		final JobID jobId = request.getPathParameter(JobIDPathParameter.class);
 		final List<TerminationModeQueryParameter.TerminationMode> terminationModes = request.getQueryParameter(TerminationModeQueryParameter.class);
 		final TerminationModeQueryParameter.TerminationMode terminationMode;

http://git-wip-us.apache.org/repos/asf/flink/blob/a5105894/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/RestfulGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/RestfulGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/RestfulGateway.java
index 28465a7..6cad0fb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/RestfulGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/RestfulGateway.java
@@ -26,6 +26,7 @@ import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.executiongraph.AccessExecutionGraph;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.messages.FlinkJobNotFoundException;
 import org.apache.flink.runtime.messages.webmonitor.ClusterOverview;
 import org.apache.flink.runtime.messages.webmonitor.MultipleJobsDetails;
@@ -47,6 +48,24 @@ import java.util.concurrent.CompletableFuture;
 public interface RestfulGateway extends RpcGateway {
 
 	/**
+	 * Cancel the given job.
+	 *
+	 * @param jobId identifying the job to cancel
+	 * @param timeout of the operation
+	 * @return A future acknowledge if the cancellation succeeded
+	 */
+	CompletableFuture<Acknowledge> cancelJob(JobID jobId, @RpcTimeout Time timeout);
+
+	/**
+	 * Stop the given job.
+	 *
+	 * @param jobId identifying the job to stop
+	 * @param timeout of the operation
+	 * @return A future acknowledge if the stopping succeeded
+	 */
+	CompletableFuture<Acknowledge> stopJob(JobID jobId, @RpcTimeout Time timeout);
+
+	/**
 	 * Requests the REST address of this {@link RpcEndpoint}.
 	 *
 	 * @param timeout for this operation

http://git-wip-us.apache.org/repos/asf/flink/blob/a5105894/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
index dfcb134..5aa3271 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
@@ -40,6 +40,7 @@ import org.apache.flink.runtime.rest.handler.job.JobExceptionsHandler;
 import org.apache.flink.runtime.rest.handler.job.JobExecutionResultHandler;
 import org.apache.flink.runtime.rest.handler.job.JobIdsHandler;
 import org.apache.flink.runtime.rest.handler.job.JobPlanHandler;
+import org.apache.flink.runtime.rest.handler.job.JobTerminationHandler;
 import org.apache.flink.runtime.rest.handler.job.JobVertexAccumulatorsHandler;
 import org.apache.flink.runtime.rest.handler.job.JobVertexBackPressureHandler;
 import org.apache.flink.runtime.rest.handler.job.JobVertexTaskManagersHandler;
@@ -78,6 +79,7 @@ import org.apache.flink.runtime.rest.messages.JobConfigHeaders;
 import org.apache.flink.runtime.rest.messages.JobExceptionsHeaders;
 import org.apache.flink.runtime.rest.messages.JobIdsWithStatusesOverviewHeaders;
 import org.apache.flink.runtime.rest.messages.JobPlanHeaders;
+import org.apache.flink.runtime.rest.messages.JobTerminationHeaders;
 import org.apache.flink.runtime.rest.messages.JobVertexAccumulatorsHeaders;
 import org.apache.flink.runtime.rest.messages.JobVertexBackPressureHeaders;
 import org.apache.flink.runtime.rest.messages.JobVertexTaskManagersHeaders;
@@ -449,6 +451,13 @@ public class WebMonitorEndpoint<T extends RestfulGateway> extends RestServerEndp
 			responseHeaders,
 			JobVertexBackPressureHeaders.getInstance());
 
+		final JobTerminationHandler jobTerminationHandler = new JobTerminationHandler(
+			restAddressFuture,
+			leaderRetriever,
+			timeout,
+			responseHeaders,
+			JobTerminationHeaders.getInstance());
+
 		final File tmpDir = restConfiguration.getTmpDir();
 
 		Optional<StaticFileServerHandler<T>> optWebContent;
@@ -495,6 +504,7 @@ public class WebMonitorEndpoint<T extends RestfulGateway> extends RestServerEndp
 		handlers.add(Tuple2.of(SubtaskCurrentAttemptDetailsHeaders.getInstance(), subtaskCurrentAttemptDetailsHandler));
 		handlers.add(Tuple2.of(JobVertexTaskManagersHeaders.getInstance(), jobVertexTaskManagersHandler));
 		handlers.add(Tuple2.of(JobVertexBackPressureHeaders.getInstance(), jobVertexBackPressureHandler));
+		handlers.add(Tuple2.of(JobTerminationHeaders.getInstance(), jobTerminationHandler));
 
 		optWebContent.ifPresent(
 			webContent -> handlers.add(Tuple2.of(WebContentHandlerSpecification.getInstance(), webContent)));

http://git-wip-us.apache.org/repos/asf/flink/blob/a5105894/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/JobVertexBackPressureHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/JobVertexBackPressureHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/JobVertexBackPressureHandlerTest.java
index b5a6ba6..6e97f6b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/JobVertexBackPressureHandlerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/JobVertexBackPressureHandlerTest.java
@@ -70,7 +70,7 @@ public class JobVertexBackPressureHandlerTest {
 
 	@Before
 	public void setUp() {
-		restfulGateway = TestingRestfulGateway.newBuilder().setRequestOeratorBackPressureStatsFunction(
+		restfulGateway = TestingRestfulGateway.newBuilder().setRequestOperatorBackPressureStatsFunction(
 			(jobId, jobVertexId) -> {
 				if (jobId.equals(TEST_JOB_ID_BACK_PRESSURE_STATS_AVAILABLE)) {
 					return CompletableFuture.completedFuture(OperatorBackPressureStatsResponse.of(new OperatorBackPressureStats(

http://git-wip-us.apache.org/repos/asf/flink/blob/a5105894/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingRestfulGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingRestfulGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingRestfulGateway.java
index 8b9f671..dc88f73 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingRestfulGateway.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingRestfulGateway.java
@@ -26,6 +26,7 @@ import org.apache.flink.runtime.concurrent.FutureUtils;
 import org.apache.flink.runtime.executiongraph.AccessExecutionGraph;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.messages.webmonitor.ClusterOverview;
 import org.apache.flink.runtime.messages.webmonitor.MultipleJobsDetails;
 import org.apache.flink.runtime.rest.handler.legacy.backpressure.OperatorBackPressureStatsResponse;
@@ -42,6 +43,8 @@ import java.util.function.Supplier;
  */
 public class TestingRestfulGateway implements RestfulGateway {
 
+	static final Function<JobID, CompletableFuture<Acknowledge>> DEFAULT_CANCEL_JOB_FUNCTION = jobId -> CompletableFuture.completedFuture(Acknowledge.get());
+	static final Function<JobID, CompletableFuture<Acknowledge>> DEFAULT_STOP_JOB_FUNCTION = jobId -> CompletableFuture.completedFuture(Acknowledge.get());
 	static final Function<JobID, CompletableFuture<? extends AccessExecutionGraph>> DEFAULT_REQUEST_JOB_FUNCTION = jobId -> FutureUtils.completedExceptionally(new UnsupportedOperationException());
 	static final Function<JobID, CompletableFuture<JobStatus>> DEFAULT_REQUEST_JOB_STATUS_FUNCTION = jobId -> FutureUtils.completedExceptionally(new UnsupportedOperationException());
 	static final Supplier<CompletableFuture<MultipleJobsDetails>> DEFAULT_REQUEST_MULTIPLE_JOB_DETAILS_SUPPLIER = () -> CompletableFuture.completedFuture(new MultipleJobsDetails(Collections.emptyList()));
@@ -57,6 +60,10 @@ public class TestingRestfulGateway implements RestfulGateway {
 
 	protected String restAddress;
 
+	protected Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction;
+
+	protected Function<JobID, CompletableFuture<Acknowledge>> stopJobFunction;
+
 	protected Function<JobID, CompletableFuture<? extends AccessExecutionGraph>> requestJobFunction;
 
 	protected Function<JobID, CompletableFuture<JobStatus>> requestJobStatusFunction;
@@ -76,6 +83,8 @@ public class TestingRestfulGateway implements RestfulGateway {
 			LOCALHOST,
 			LOCALHOST,
 			LOCALHOST,
+			DEFAULT_CANCEL_JOB_FUNCTION,
+			DEFAULT_STOP_JOB_FUNCTION,
 			DEFAULT_REQUEST_JOB_FUNCTION,
 			DEFAULT_REQUEST_JOB_STATUS_FUNCTION,
 			DEFAULT_REQUEST_MULTIPLE_JOB_DETAILS_SUPPLIER,
@@ -89,6 +98,8 @@ public class TestingRestfulGateway implements RestfulGateway {
 			String address,
 			String hostname,
 			String restAddress,
+			Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction,
+			Function<JobID, CompletableFuture<Acknowledge>> stopJobFunction,
 			Function<JobID, CompletableFuture<? extends AccessExecutionGraph>> requestJobFunction,
 			Function<JobID, CompletableFuture<JobStatus>> requestJobStatusFunction,
 			Supplier<CompletableFuture<MultipleJobsDetails>> requestMultipleJobDetailsSupplier,
@@ -99,6 +110,8 @@ public class TestingRestfulGateway implements RestfulGateway {
 		this.address = address;
 		this.hostname = hostname;
 		this.restAddress = restAddress;
+		this.cancelJobFunction = cancelJobFunction;
+		this.stopJobFunction = stopJobFunction;
 		this.requestJobFunction = requestJobFunction;
 		this.requestJobStatusFunction = requestJobStatusFunction;
 		this.requestMultipleJobDetailsSupplier = requestMultipleJobDetailsSupplier;
@@ -109,6 +122,16 @@ public class TestingRestfulGateway implements RestfulGateway {
 	}
 
 	@Override
+	public CompletableFuture<Acknowledge> cancelJob(JobID jobId, Time timeout) {
+		return cancelJobFunction.apply(jobId);
+	}
+
+	@Override
+	public CompletableFuture<Acknowledge> stopJob(JobID jobId, Time timeout) {
+		return stopJobFunction.apply(jobId);
+	}
+
+	@Override
 	public CompletableFuture<String> requestRestAddress(Time timeout) {
 		return CompletableFuture.completedFuture(restAddress);
 	}
@@ -169,22 +192,26 @@ public class TestingRestfulGateway implements RestfulGateway {
 		private String address = LOCALHOST;
 		private String hostname = LOCALHOST;
 		private String restAddress = LOCALHOST;
+		private Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction;
+		private Function<JobID, CompletableFuture<Acknowledge>> stopJobFunction;
 		private Function<JobID, CompletableFuture<? extends AccessExecutionGraph>> requestJobFunction;
 		private Function<JobID, CompletableFuture<JobStatus>> requestJobStatusFunction;
 		private Supplier<CompletableFuture<MultipleJobsDetails>> requestMultipleJobDetailsSupplier;
 		private Supplier<CompletableFuture<ClusterOverview>> requestClusterOverviewSupplier;
 		private Supplier<CompletableFuture<Collection<String>>> requestMetricQueryServicePathsSupplier;
 		private Supplier<CompletableFuture<Collection<Tuple2<ResourceID, String>>>> requestTaskManagerMetricQueryServicePathsSupplier;
-		private BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOeratorBackPressureStatsFunction;
+		private BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction;
 
 		public Builder() {
+			cancelJobFunction = DEFAULT_CANCEL_JOB_FUNCTION;
+			stopJobFunction = DEFAULT_STOP_JOB_FUNCTION;
 			requestJobFunction = DEFAULT_REQUEST_JOB_FUNCTION;
 			requestJobStatusFunction = DEFAULT_REQUEST_JOB_STATUS_FUNCTION;
 			requestMultipleJobDetailsSupplier = DEFAULT_REQUEST_MULTIPLE_JOB_DETAILS_SUPPLIER;
 			requestClusterOverviewSupplier = DEFAULT_REQUEST_CLUSTER_OVERVIEW_SUPPLIER;
 			requestMetricQueryServicePathsSupplier = DEFAULT_REQUEST_METRIC_QUERY_SERVICE_PATHS_SUPPLIER;
 			requestTaskManagerMetricQueryServicePathsSupplier = DEFAULT_REQUEST_TASK_MANAGER_METRIC_QUERY_SERVICE_PATHS_SUPPLIER;
-			requestOeratorBackPressureStatsFunction = DEFAULT_REQUEST_OPERATOR_BACK_PRESSURE_STATS_SUPPLIER;
+			requestOperatorBackPressureStatsFunction = DEFAULT_REQUEST_OPERATOR_BACK_PRESSURE_STATS_SUPPLIER;
 		}
 
 		public Builder setAddress(String address) {
@@ -232,13 +259,35 @@ public class TestingRestfulGateway implements RestfulGateway {
 			return this;
 		}
 
-		public Builder setRequestOeratorBackPressureStatsFunction(BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOeratorBackPressureStatsFunction) {
-			this.requestOeratorBackPressureStatsFunction = requestOeratorBackPressureStatsFunction;
+		public Builder setRequestOperatorBackPressureStatsFunction(BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOeratorBackPressureStatsFunction) {
+			this.requestOperatorBackPressureStatsFunction = requestOeratorBackPressureStatsFunction;
+			return this;
+		}
+
+		public Builder setCancelJobFunction(Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction) {
+			this.cancelJobFunction = cancelJobFunction;
+			return this;
+		}
+
+		public Builder setStopJobFunction(Function<JobID, CompletableFuture<Acknowledge>> stopJobFunction) {
+			this.stopJobFunction = stopJobFunction;
 			return this;
 		}
 
 		public TestingRestfulGateway build() {
-			return new TestingRestfulGateway(address, hostname, restAddress, requestJobFunction, requestJobStatusFunction, requestMultipleJobDetailsSupplier, requestClusterOverviewSupplier, requestMetricQueryServicePathsSupplier, requestTaskManagerMetricQueryServicePathsSupplier, requestOeratorBackPressureStatsFunction);
+			return new TestingRestfulGateway(
+				address,
+				hostname,
+				restAddress,
+				cancelJobFunction,
+				stopJobFunction,
+				requestJobFunction,
+				requestJobStatusFunction,
+				requestMultipleJobDetailsSupplier,
+				requestClusterOverviewSupplier,
+				requestMetricQueryServicePathsSupplier,
+				requestTaskManagerMetricQueryServicePathsSupplier,
+				requestOperatorBackPressureStatsFunction);
 		}
 	}
 }


[09/22] flink git commit: [FLINK-8609] [flip6] Enable Flip-6 job mode in CliFrontend

Posted by tr...@apache.org.
[FLINK-8609] [flip6] Enable Flip-6 job mode in CliFrontend

This commit allows to deploy detached job mode clusters via the
CliFrontend. In order to do that, it first extracts the JobGraph
from the PackagedProgram and then uses the ClusterDescriptor to
deploy the job mode cluster.

This closes #5432.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c9bc9749
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c9bc9749
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c9bc9749

Branch: refs/heads/master
Commit: c9bc97490ed8a658f03cb65a776fba2053b2d46f
Parents: 84418d0
Author: Till Rohrmann <tr...@apache.org>
Authored: Tue Feb 6 16:47:28 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Thu Feb 15 08:44:19 2018 +0100

----------------------------------------------------------------------
 .../apache/flink/client/cli/CliFrontend.java    | 166 +++++++++++++++----
 1 file changed, 130 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/c9bc9749/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
index 560bc6f..23e82bc 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java
@@ -23,10 +23,13 @@ import org.apache.flink.api.common.InvalidProgramException;
 import org.apache.flink.api.common.JobExecutionResult;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.JobSubmissionResult;
+import org.apache.flink.api.common.Plan;
 import org.apache.flink.api.common.accumulators.AccumulatorHelper;
 import org.apache.flink.client.deployment.ClusterDescriptor;
 import org.apache.flink.client.deployment.ClusterSpecification;
 import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.client.program.JobWithJars;
+import org.apache.flink.client.program.OptimizerPlanEnvironment;
 import org.apache.flink.client.program.PackagedProgram;
 import org.apache.flink.client.program.ProgramInvocationException;
 import org.apache.flink.client.program.ProgramMissingJobException;
@@ -37,6 +40,7 @@ import org.apache.flink.configuration.CoreOptions;
 import org.apache.flink.configuration.GlobalConfiguration;
 import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
 import org.apache.flink.optimizer.DataStatistics;
 import org.apache.flink.optimizer.Optimizer;
 import org.apache.flink.optimizer.costs.DefaultCostEstimator;
@@ -44,9 +48,11 @@ import org.apache.flink.optimizer.plan.FlinkPlan;
 import org.apache.flink.optimizer.plan.OptimizedPlan;
 import org.apache.flink.optimizer.plan.StreamingPlan;
 import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.client.JobStatusMessage;
 import org.apache.flink.runtime.concurrent.FutureUtils;
+import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.messages.JobManagerMessages;
@@ -68,6 +74,7 @@ import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -113,6 +120,8 @@ public class CliFrontend {
 
 	private final int defaultParallelism;
 
+	private final boolean flip6;
+
 	public CliFrontend(
 			Configuration configuration,
 			List<CustomCommandLine<?>> customCommandLines) throws Exception {
@@ -135,6 +144,8 @@ public class CliFrontend {
 
 		this.clientTimeout = AkkaUtils.getClientTimeout(this.configuration);
 		this.defaultParallelism = configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM);
+
+		this.flip6 = CoreOptions.FLIP6_MODE.equalsIgnoreCase(configuration.getString(CoreOptions.MODE));
 	}
 
 	// --------------------------------------------------------------------------------------------
@@ -195,7 +206,11 @@ public class CliFrontend {
 
 		final CustomCommandLine<?> customCommandLine = getActiveCustomCommandLine(commandLine);
 
-		runProgram(customCommandLine, commandLine, runOptions, program);
+		try {
+			runProgram(customCommandLine, commandLine, runOptions, program);
+		} finally {
+			program.deleteExtractedLibraries();
+		}
 	}
 
 	private <T> void runProgram(
@@ -210,51 +225,72 @@ public class CliFrontend {
 
 			final ClusterClient<T> client;
 
-			if (clusterId != null) {
-				client = clusterDescriptor.retrieve(clusterId);
-			} else {
-				final ClusterSpecification clusterSpecification = customCommandLine.getClusterSpecification(commandLine);
-				client = clusterDescriptor.deploySessionCluster(clusterSpecification);
-			}
+			// directly deploy the job if the cluster is started in job mode and detached
+			if (flip6 && clusterId == null && runOptions.getDetachedMode()) {
+				int parallelism = runOptions.getParallelism() == -1 ? defaultParallelism : runOptions.getParallelism();
 
-			try {
-				client.setPrintStatusDuringExecution(runOptions.getStdoutLogging());
-				client.setDetached(runOptions.getDetachedMode());
-				LOG.debug("Client slots is set to {}", client.getMaxSlots());
-
-				LOG.debug(runOptions.getSavepointRestoreSettings().toString());
-
-				int userParallelism = runOptions.getParallelism();
-				LOG.debug("User parallelism is set to {}", userParallelism);
-				if (client.getMaxSlots() != -1 && userParallelism == -1) {
-					logAndSysout("Using the parallelism provided by the remote cluster ("
-						+ client.getMaxSlots() + "). "
-						+ "To use another parallelism, set it at the ./bin/flink client.");
-					userParallelism = client.getMaxSlots();
-				} else if (ExecutionConfig.PARALLELISM_DEFAULT == userParallelism) {
-					userParallelism = defaultParallelism;
-				}
+				final JobGraph jobGraph = createJobGraph(configuration, program, parallelism);
 
-				executeProgram(program, client, userParallelism);
-			} finally {
-				if (clusterId == null && !client.isDetached()) {
-					// terminate the cluster only if we have started it before and if it's not detached
-					try {
-						clusterDescriptor.terminateCluster(client.getClusterId());
-					} catch (FlinkException e) {
-						LOG.info("Could not properly terminate the Flink cluster.", e);
-					}
-				}
+				final ClusterSpecification clusterSpecification = customCommandLine.getClusterSpecification(commandLine);
+				client = clusterDescriptor.deployJobCluster(
+					clusterSpecification,
+					jobGraph,
+					runOptions.getDetachedMode());
+
+				logAndSysout("Job has been submitted with JobID " + jobGraph.getJobID());
 
 				try {
 					client.shutdown();
 				} catch (Exception e) {
 					LOG.info("Could not properly shut down the client.", e);
 				}
+			} else {
+				if (clusterId != null) {
+					client = clusterDescriptor.retrieve(clusterId);
+				} else {
+					// also in job mode we have to deploy a session cluster because the job
+					// might consist of multiple parts (e.g. when using collect)
+					final ClusterSpecification clusterSpecification = customCommandLine.getClusterSpecification(commandLine);
+					client = clusterDescriptor.deploySessionCluster(clusterSpecification);
+				}
+
+				try {
+					client.setPrintStatusDuringExecution(runOptions.getStdoutLogging());
+					client.setDetached(runOptions.getDetachedMode());
+					LOG.debug("Client slots is set to {}", client.getMaxSlots());
+
+					LOG.debug("{}", runOptions.getSavepointRestoreSettings());
+
+					int userParallelism = runOptions.getParallelism();
+					LOG.debug("User parallelism is set to {}", userParallelism);
+					if (client.getMaxSlots() != -1 && userParallelism == -1) {
+						logAndSysout("Using the parallelism provided by the remote cluster ("
+							+ client.getMaxSlots() + "). "
+							+ "To use another parallelism, set it at the ./bin/flink client.");
+						userParallelism = client.getMaxSlots();
+					} else if (ExecutionConfig.PARALLELISM_DEFAULT == userParallelism) {
+						userParallelism = defaultParallelism;
+					}
+
+					executeProgram(program, client, userParallelism);
+				} finally {
+					if (clusterId == null && !client.isDetached()) {
+						// terminate the cluster only if we have started it before and if it's not detached
+						try {
+							clusterDescriptor.terminateCluster(client.getClusterId());
+						} catch (FlinkException e) {
+							LOG.info("Could not properly terminate the Flink cluster.", e);
+						}
+					}
+
+					try {
+						client.shutdown();
+					} catch (Exception e) {
+						LOG.info("Could not properly shut down the client.", e);
+					}
+				}
 			}
 		} finally {
-			program.deleteExtractedLibraries();
-
 			try {
 				clusterDescriptor.close();
 			} catch (Exception e) {
@@ -1081,6 +1117,64 @@ public class CliFrontend {
 		return customCommandLines;
 	}
 
+	/**
+	 * Creates a {@link JobGraph} from the given {@link PackagedProgram}.
+	 *
+	 * @param configuration to use for the optimizer and job graph generator
+	 * @param packagedProgram to extract the JobGraph from
+	 * @param defaultParallelism for the JobGraph
+	 * @return JobGraph extracted from the PackagedProgram
+	 * @throws ProgramInvocationException if the JobGraph generation failed
+	 */
+	private static JobGraph createJobGraph(Configuration configuration, PackagedProgram packagedProgram, int defaultParallelism) throws ProgramInvocationException {
+		Thread.currentThread().setContextClassLoader(packagedProgram.getUserCodeClassLoader());
+		final Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), configuration);
+		final FlinkPlan flinkPlan;
+
+		if (packagedProgram.isUsingProgramEntryPoint()) {
+
+			final JobWithJars jobWithJars = packagedProgram.getPlanWithJars();
+
+			final Plan plan = jobWithJars.getPlan();
+
+			if (plan.getDefaultParallelism() <= 0) {
+				plan.setDefaultParallelism(defaultParallelism);
+			}
+
+			flinkPlan = optimizer.compile(jobWithJars.getPlan());
+		} else if (packagedProgram.isUsingInteractiveMode()) {
+			final OptimizerPlanEnvironment optimizerPlanEnvironment = new OptimizerPlanEnvironment(optimizer);
+
+			optimizerPlanEnvironment.setParallelism(defaultParallelism);
+
+			flinkPlan = optimizerPlanEnvironment.getOptimizedPlan(packagedProgram);
+		} else {
+			throw new ProgramInvocationException("PackagedProgram does not have a valid invocation mode.");
+		}
+
+		final JobGraph jobGraph;
+
+		if (flinkPlan instanceof StreamingPlan) {
+			jobGraph = ((StreamingPlan) flinkPlan).getJobGraph();
+			jobGraph.setSavepointRestoreSettings(packagedProgram.getSavepointSettings());
+		} else {
+			final JobGraphGenerator jobGraphGenerator = new JobGraphGenerator(configuration);
+			jobGraph = jobGraphGenerator.compileJobGraph((OptimizedPlan) flinkPlan);
+		}
+
+		for (URL url : packagedProgram.getAllLibraries()) {
+			try {
+				jobGraph.addJar(new Path(url.toURI()));
+			} catch (URISyntaxException e) {
+				throw new ProgramInvocationException("Invalid URL for jar file: " + url + '.', e);
+			}
+		}
+
+		jobGraph.setClasspaths(packagedProgram.getClasspaths());
+
+		return jobGraph;
+	}
+
 	// --------------------------------------------------------------------------------------------
 	//  Custom command-line
 	// --------------------------------------------------------------------------------------------


[04/22] flink git commit: [FLINK-8605] [rest] Enable job cancellation from the web UI

Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime-web/web-dashboard/web/js/index.js
----------------------------------------------------------------------
diff --git a/flink-runtime-web/web-dashboard/web/js/index.js b/flink-runtime-web/web-dashboard/web/js/index.js
index 6e51eb6..630b064 100644
--- a/flink-runtime-web/web-dashboard/web/js/index.js
+++ b/flink-runtime-web/web-dashboard/web/js/index.js
@@ -1,2 +1,2 @@
-angular.module("flinkApp",["ui.router","angularMoment","dndLists"]).run(["$rootScope",function(e){return e.sidebarVisible=!1,e.showSidebar=function(){return e.sidebarVisible=!e.sidebarVisible,e.sidebarClass="force-show"}}]).value("flinkConfig",{jobServer:"","refresh-interval":1e4}).value("watermarksConfig",{noWatermark:-0x8000000000000000}).run(["JobsService","MainService","flinkConfig","$interval",function(e,t,r,n){return t.loadConfig().then(function(t){return angular.extend(r,t),e.listJobs(),n(function(){return e.listJobs()},r["refresh-interval"])})}]).config(["$uiViewScrollProvider",function(e){return e.useAnchorScroll()}]).run(["$rootScope","$state",function(e,t){return e.$on("$stateChangeStart",function(e,r,n,i){if(r.redirectTo)return e.preventDefault(),t.go(r.redirectTo,n)})}]).config(["$stateProvider","$urlRouterProvider",function(e,t){return e.state("overview",{url:"/overview",views:{main:{templateUrl:"partials/overview.html",controller:"OverviewController"}}}).state("runnin
 g-jobs",{url:"/running-jobs",views:{main:{templateUrl:"partials/jobs/running-jobs.html",controller:"RunningJobsController"}}}).state("completed-jobs",{url:"/completed-jobs",views:{main:{templateUrl:"partials/jobs/completed-jobs.html",controller:"CompletedJobsController"}}}).state("single-job",{url:"/jobs/{jobid}","abstract":!0,views:{main:{templateUrl:"partials/jobs/job.html",controller:"SingleJobController"}}}).state("single-job.plan",{url:"",redirectTo:"single-job.plan.subtasks",views:{details:{templateUrl:"partials/jobs/job.plan.html",controller:"JobPlanController"}}}).state("single-job.plan.subtasks",{url:"",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.subtasks.html",controller:"JobPlanSubtasksController"}}}).state("single-job.plan.metrics",{url:"/metrics",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.metrics.html",controller:"JobPlanMetricsController"}}}).state("single-job.plan.watermarks",{url:"/watermarks",views:{"node-details":{
 templateUrl:"partials/jobs/job.plan.node-list.watermarks.html"}}}).state("single-job.plan.accumulators",{url:"/accumulators",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.accumulators.html",controller:"JobPlanAccumulatorsController"}}}).state("single-job.plan.checkpoints",{url:"/checkpoints",redirectTo:"single-job.plan.checkpoints.overview",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.checkpoints.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.overview",{url:"/overview",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.overview.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.summary",{url:"/summary",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.summary.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.history",{url:"/history",views:{"checkpoints-view":{template
 Url:"partials/jobs/job.plan.node.checkpoints.history.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.config",{url:"/config",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.config.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.details",{url:"/details/{checkpointId}",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.details.html",controller:"JobPlanCheckpointDetailsController"}}}).state("single-job.plan.backpressure",{url:"/backpressure",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.backpressure.html",controller:"JobPlanBackPressureController"}}}).state("single-job.timeline",{url:"/timeline",views:{details:{templateUrl:"partials/jobs/job.timeline.html"}}}).state("single-job.timeline.vertex",{url:"/{vertexId}",views:{vertex:{templateUrl:"partials/jobs/job.timeline.vertex.html",controller:"JobTimelineVertexController"}}}).sta
 te("single-job.exceptions",{url:"/exceptions",views:{details:{templateUrl:"partials/jobs/job.exceptions.html",controller:"JobExceptionsController"}}}).state("single-job.config",{url:"/config",views:{details:{templateUrl:"partials/jobs/job.config.html"}}}).state("all-manager",{url:"/taskmanagers",views:{main:{templateUrl:"partials/taskmanager/index.html",controller:"AllTaskManagersController"}}}).state("single-manager",{url:"/taskmanager/{taskmanagerid}","abstract":!0,views:{main:{templateUrl:"partials/taskmanager/taskmanager.html",controller:"SingleTaskManagerController"}}}).state("single-manager.metrics",{url:"/metrics",views:{details:{templateUrl:"partials/taskmanager/taskmanager.metrics.html"}}}).state("single-manager.stdout",{url:"/stdout",views:{details:{templateUrl:"partials/taskmanager/taskmanager.stdout.html",controller:"SingleTaskManagerStdoutController"}}}).state("single-manager.log",{url:"/log",views:{details:{templateUrl:"partials/taskmanager/taskmanager.log.html",contro
 ller:"SingleTaskManagerLogsController"}}}).state("jobmanager",{url:"/jobmanager",views:{main:{templateUrl:"partials/jobmanager/index.html"}}}).state("jobmanager.config",{url:"/config",views:{details:{templateUrl:"partials/jobmanager/config.html",controller:"JobManagerConfigController"}}}).state("jobmanager.stdout",{url:"/stdout",views:{details:{templateUrl:"partials/jobmanager/stdout.html",controller:"JobManagerStdoutController"}}}).state("jobmanager.log",{url:"/log",views:{details:{templateUrl:"partials/jobmanager/log.html",controller:"JobManagerLogsController"}}}).state("submit",{url:"/submit",views:{main:{templateUrl:"partials/submit.html",controller:"JobSubmitController"}}}),t.otherwise("/overview")}]),angular.module("flinkApp").directive("bsLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getLab
 elClass=function(){return"label label-"+e.translateLabelState(n.status)}}}}]).directive("bpLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getBackPressureLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getBackPressureLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getBackPressureLabelClass=function(){return"label label-"+e.translateBackPressureLabelState(n.status)}}}}]).directive("indicatorPrimary",["JobsService",function(e){return{replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<i title='{{status}}' ng-class='getLabelClass()' />",link:function(t,r,n){return t.getLabelClass=function(){return"fa fa-circle indicator indicator-"+e.translateLabelState(n.status)}}}}]).directive("tableProperty",function(){return{replace:!0,scope:{value:"="},template:"<td title=\"{{value || 'None'}}\">{{value || 'None'}}</td>"}}),angular.module("flinkApp").filter("amDurationFormatExtended",["angularMomentConfig",
 function(e){var t;return t=function(e,t,r){return"undefined"==typeof e||null===e?"":moment.duration(e,t).format(r,{trim:!1})},t.$stateful=e.statefulFilters,t}]).filter("humanizeDuration",function(){return function(e,t){var r,n,i,o,a,s;return"undefined"==typeof e||null===e?"":(o=e%1e3,s=Math.floor(e/1e3),a=s%60,s=Math.floor(s/60),i=s%60,s=Math.floor(s/60),n=s%24,s=Math.floor(s/24),r=s,0===r?0===n?0===i?0===a?o+"ms":a+"s ":i+"m "+a+"s":t?n+"h "+i+"m":n+"h "+i+"m "+a+"s":t?r+"d "+n+"h":r+"d "+n+"h "+i+"m "+a+"s")}}).filter("limit",function(){return function(e){return e.length>73&&(e=e.substring(0,35)+"..."+e.substring(e.length-35,e.length)),e}}).filter("humanizeText",function(){return function(e){return e?e.replace(/&gt;/g,">").replace(/<br\/>/g,""):""}}).filter("humanizeBytes",function(){return function(e){var t,r;return r=["B","KB","MB","GB","TB","PB","EB"],t=function(e,n){var i;return i=Math.pow(1024,n),e<i?(e/i).toFixed(2)+" "+r[n]:e<1e3*i?(e/i).toPrecision(3)+" "+r[n]:t(e,n+1)},"u
 ndefined"==typeof e||null===e?"":e<1e3?e+" B":t(e,1)}}).filter("toLocaleString",function(){return function(e){return e.toLocaleString()}}).filter("toUpperCase",function(){return function(e){return e.toUpperCase()}}).filter("percentage",function(){return function(e){return(100*e).toFixed(0)+"%"}}).filter("humanizeWatermark",["watermarksConfig",function(e){return function(t){return isNaN(t)||t<=e.noWatermark?"No Watermark":t}}]).filter("increment",function(){return function(e){return parseInt(e)+1}}).filter("humanizeChartNumeric",["humanizeBytesFilter","humanizeDurationFilter",function(e,t){return function(r,n){var i;return i="",null!==r&&(i=/bytes/i.test(n.id)&&/persecond/i.test(n.id)?e(r)+" / s":/bytes/i.test(n.id)?e(r):/persecond/i.test(n.id)?r+" / s":/time/i.test(n.id)||/latency/i.test(n.id)?t(r,!0):r),i}}]).filter("humanizeChartNumericTitle",["humanizeDurationFilter",function(e){return function(t,r){var n;return n="",null!==t&&(n=/bytes/i.test(r.id)&&/persecond/i.test(r.id)?t+" B
 ytes / s":/bytes/i.test(r.id)?t+" Bytes":/persecond/i.test(r.id)?t+" / s":/time/i.test(r.id)||/latency/i.test(r.id)?e(t,!1):t),n}}]).filter("searchMetrics",function(){return function(e,t){var r,n;return n=new RegExp(t,"gi"),function(){var t,i,o;for(o=[],t=0,i=e.length;t<i;t++)r=e[t],r.id.match(n)&&o.push(r);return o}()}}),angular.module("flinkApp").service("MainService",["$http","flinkConfig","$q",function(e,t,r){return this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"config").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this}]),angular.module("flinkApp").controller("RunningJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("running")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("CompletedJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){r
 eturn e.jobObserver=function(){return e.jobs=n.getJobs("finished")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("SingleJobController",["$scope","$state","$stateParams","JobsService","MetricsService","$rootScope","flinkConfig","$interval","$q","watermarksConfig",function(e,t,r,n,i,o,a,s,l,u){var c,d;return e.jobid=r.jobid,e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats={},d=s(function(){return n.loadJob(r.jobid).then(function(t){return e.job=t,e.$broadcast("reload")})},a["refresh-interval"]),e.$on("$destroy",function(){return e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats=null,s.cancel(d)}),e.cancelJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Cancelling..."),n.cancelJob(r.jobid).then(function(e){return{}})},e.stopJob=function(e){return angular.element(e.current
 Target).removeClass("btn").removeClass("btn-default").html("Stopping..."),n.stopJob(r.jobid).then(function(e){return{}})},n.loadJob(r.jobid).then(function(t){return e.job=t,e.vertices=t.vertices,e.plan=t.plan,i.setupMetrics(r.jobid,t.vertices)}),c=function(t){var r,n,o,a;return o=function(t){return function(t){var r,n,o,a;return r=l.defer(),o=e.job.jid,a=function(){var e,r,i;for(i=[],n=e=0,r=t.parallelism-1;0<=r?e<=r:e>=r;n=0<=r?++e:--e)i.push(n+".currentLowWatermark");return i}(),i.getMetrics(o,t.id,a).then(function(e){var t,n,i,o,a,s,l;i=NaN,l={},o=e.values;for(t in o)s=o[t],a=t.replace(".currentLowWatermark",""),l[a]=s,(isNaN(i)||s<i)&&(i=s);return n=!isNaN(i)&&i>u.noWatermark?i:NaN,r.resolve({lowWatermark:n,watermarks:l})}),r.promise}}(this),r=l.defer(),a={},n=t.length,angular.forEach(t,function(e){return function(e,t){var i;return i=e.id,o(e).then(function(e){if(a[i]=e,t>=n-1)return r.resolve(a)})}}(this)),r.promise},e.hasWatermark=function(t){return e.watermarks[t]&&!isNaN(e.w
 atermarks[t].lowWatermark)},e.$watch("plan",function(t){if(t)return c(t.nodes).then(function(t){return e.watermarks=t})}),e.$on("reload",function(){if(e.plan)return c(e.plan.nodes).then(function(t){return e.watermarks=t})})}]).controller("JobPlanController",["$scope","$state","$stateParams","$window","JobsService",function(e,t,r,n,i){return e.nodeid=null,e.nodeUnfolded=!1,e.stateList=i.stateList(),e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null,e.$broadcast("reload"),e.$broadcast("node:change",e.nodeid)):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null)},e.deactivateNode=function(){return e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null},e.toggleFold=function(){return e.nodeUnfolded=!e.nodeUnfolded}}]).controller("JobPlanSubtasksController",["$scope","JobsService",fun
 ction(e,t){var r;return e.aggregate=!1,r=function(){return e.aggregate?t.getTaskManagers(e.nodeid).then(function(t){return e.taskmanagers=t}):t.getSubtasks(e.nodeid).then(function(t){return e.subtasks=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanAccumulatorsController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getAccumulators(e.nodeid).then(function(t){return e.accumulators=t.main,e.subtaskAccumulators=t.subtasks})},!e.nodeid||e.vertex&&e.vertex.accumulators||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanCheckpointsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return e.checkpointDetails={},e.checkpointDetails.id=-1,n.getCheckpointConfig().then(function(t){return e.checkpointConfig=t}),i=function(){return n.getCheckpointStats().then(function(t){if(null!==t)return e.checkpointStats=t})},i(),e.$on("reload",function(
 e){return i()})}]).controller("JobPlanCheckpointDetailsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i,o;return e.subtaskDetails={},e.checkpointDetails.id=r.checkpointId,i=function(t){return n.getCheckpointDetails(t).then(function(t){return null!==t?e.checkpoint=t:e.unknown_checkpoint=!0})},o=function(t,r){return n.getCheckpointSubtaskDetails(t,r).then(function(t){if(null!==t)return e.subtaskDetails[r]=t})},i(r.checkpointId),e.nodeid&&o(r.checkpointId,e.nodeid),e.$on("reload",function(t){if(i(r.checkpointId),e.nodeid)return o(r.checkpointId,e.nodeid)}),e.$on("$destroy",function(){return e.checkpointDetails.id=-1})}]).controller("JobPlanBackPressureController",["$scope","JobsService",function(e,t){var r;return r=function(){if(e.now=Date.now(),e.nodeid)return t.getOperatorBackPressure(e.nodeid).then(function(t){return e.backPressureOperatorStats[e.nodeid]=t})},r(),e.$on("reload",function(e){return r()})}]).controller("JobTimelineVertexController",["
 $scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return i=function(){return n.getVertex(r.vertexId).then(function(t){return e.vertex=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobExceptionsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return n.loadExceptions().then(function(t){return e.exceptions=t})}]).controller("JobPropertiesController",["$scope","JobsService",function(e,t){return e.changeNode=function(r){return r!==e.nodeid?(e.nodeid=r,t.getNode(r).then(function(t){return e.node=t})):(e.nodeid=null,e.node=null)}}]).controller("JobPlanMetricsController",["$scope","JobsService","MetricsService",function(e,t,r){var n,i;if(e.dragging=!1,e.window=r.getWindow(),e.availableMetrics=null,e.$on("$destroy",function(){return r.unRegisterObserver()}),i=function(){return t.getVertex(e.nodeid).then(function(t){return e.vertex=t}),r.getAvailableMetrics(e.jobid,e.nodeid).then(function(t){return e.availableMetrics=t.sort(n)
 ,e.metrics=r.getMetricsSetup(e.jobid,e.nodeid).names,r.registerObserver(e.jobid,e.nodeid,function(t){return e.$broadcast("metrics:data:update",t.timestamp,t.values)})})},n=function(e,t){var r,n;return r=e.id.toLowerCase(),n=t.id.toLowerCase(),r<n?-1:r>n?1:0},e.dropped=function(t,n,o,a,s){return r.orderMetrics(e.jobid,e.nodeid,o,n),e.$broadcast("metrics:refresh",o),i(),!1},e.dragStart=function(){return e.dragging=!0},e.dragEnd=function(){return e.dragging=!1},e.addMetric=function(t){return r.addMetric(e.jobid,e.nodeid,t.id),i()},e.removeMetric=function(t){return r.removeMetric(e.jobid,e.nodeid,t),i()},e.setMetricSize=function(t,n){return r.setMetricSize(e.jobid,e.nodeid,t,n),i()},e.setMetricView=function(t,n){return r.setMetricView(e.jobid,e.nodeid,t,n),i()},e.getValues=function(t){return r.getValues(e.jobid,e.nodeid,t)},e.$on("node:change",function(t,r){if(!e.dragging)return i()}),e.nodeid)return i()}]),angular.module("flinkApp").directive("vertex",["$state",function(e){return{templ
 ate:"<svg class='timeline secondary' width='0' height='0'></svg>",scope:{data:"="},link:function(e,t,r){var n,i,o;o=t.children()[0],i=t.width(),angular.element(o).attr("width",i),(n=function(e){var t,r,n;return d3.select(o).selectAll("*").remove(),n=[],angular.forEach(e.subtasks,function(e,t){var r;return r=[{label:"Scheduled",color:"#666",borderColor:"#555",starting_time:e.timestamps.SCHEDULED,ending_time:e.timestamps.DEPLOYING,type:"regular"},{label:"Deploying",color:"#aaa",borderColor:"#555",starting_time:e.timestamps.DEPLOYING,ending_time:e.timestamps.RUNNING,type:"regular"}],e.timestamps.FINISHED>0&&r.push({label:"Running",color:"#ddd",borderColor:"#555",starting_time:e.timestamps.RUNNING,ending_time:e.timestamps.FINISHED,type:"regular"}),n.push({label:"("+e.subtask+") "+e.host,times:r})}),t=d3.timeline().stack().tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("single").labelFormat(function(e){return e}).margin({left:100,right:0,top:0,bottom:0}).itemHeight(30).relat
 iveTime(),r=d3.select(o).datum(n).call(t)})(e.data)}}}]).directive("timeline",["$state",function(e){return{template:"<svg class='timeline' width='0' height='0'></svg>",scope:{vertices:"=",jobid:"="},link:function(t,r,n){var i,o,a,s;a=r.children()[0],o=r.width(),angular.element(a).attr("width",o),s=function(e){return e.replace("&gt;",">")},i=function(r){var n,i,o;return d3.select(a).selectAll("*").remove(),o=[],angular.forEach(r,function(e){if(e["start-time"]>-1)return"scheduled"===e.type?o.push({times:[{label:s(e.name),color:"#cccccc",borderColor:"#555555",starting_time:e["start-time"],ending_time:e["end-time"],type:e.type}]}):o.push({times:[{label:s(e.name),color:"#d9f1f7",borderColor:"#62cdea",starting_time:e["start-time"],ending_time:e["end-time"],link:e.id,type:e.type}]})}),n=d3.timeline().stack().click(function(r,n,i){if(r.link)return e.go("single-job.timeline.vertex",{jobid:t.jobid,vertexId:r.link})}).tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("main").margin({
 left:0,right:0,top:0,bottom:0}).itemHeight(30).showBorderLine().showHourTimeline(),i=d3.select(a).datum(o).call(n)},t.$watch(n.vertices,function(e){if(e)return i(e)})}}}]).directive("split",function(){return{compile:function(e,t){return Split(e.children(),{sizes:[50,50],direction:"vertical"})}}}).directive("jobPlan",["$timeout",function(e){return{template:"<svg class='graph'><g /></svg> <svg class='tmp' width='1' height='1'><g /></svg> <div class='btn-group zoom-buttons'> <a class='btn btn-default zoom-in' ng-click='zoomIn()'><i class='fa fa-plus' /></a> <a class='btn btn-default zoom-out' ng-click='zoomOut()'><i class='fa fa-minus' /></a> </div>",scope:{plan:"=",watermarks:"=",setNode:"&"},link:function(e,t,r){var n,i,o,a,s,l,u,c,d,f,p,m,g,h,b,v,k,j,S,w,C,$,M,y,J;p=null,C=d3.behavior.zoom(),J=[],h=r.jobid,S=t.children()[0],j=t.children().children()[0],w=t.children()[1],l=d3.select(S),u=d3.select(j),c=d3.select(w),n=t.width(),angular.element(t.children()[0]).width(n),v=0,b=0,e.zoomI
 n=function(){var e,t,r;if(C.scale()<2.99)return e=C.translate(),t=e[0]*(C.scale()+.1/C.scale()),r=e[1]*(C.scale()+.1/C.scale()),C.scale(C.scale()+.1),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},e.zoomOut=function(){var e,t,r;if(C.scale()>.31)return C.scale(C.scale()-.1),e=C.translate(),t=e[0]*(C.scale()-.1/C.scale()),r=e[1]*(C.scale()-.1/C.scale()),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},o=function(e){var t;return t="",null==e.ship_strategy&&null==e.local_strategy||(t+="<div class='edge-label'>",null!=e.ship_strategy&&(t+=e.ship_strategy),void 0!==e.temp_mode&&(t+=" ("+e.temp_mode+")"),void 0!==e.local_strategy&&(t+=",<br>"+e.local_strategy),t+="</div>"),t},g=function(e){return"partialSolution"===e||"nextPartialSolution"===e||"workset"===e||"nextWorkset"===e||"solutionSet"===e||"solutionDelta"===e},m=function(e,t){return"mirror"===t?"node-mi
 rror":g(t)?"node-iteration":"node-normal"},a=function(e,t,r,n){var i,o;return i="<div href='#/jobs/"+h+"/vertex/"+e.id+"' class='node-label "+m(e,t)+"'>",i+="mirror"===t?"<h3 class='node-name'>Mirror of "+e.operator+"</h3>":"<h3 class='node-name'>"+e.operator+"</h3>",""===e.description?i+="":(o=e.description,o=y(o),i+="<h4 class='step-name'>"+o+"</h4>"),null!=e.step_function?i+=f(e.id,r,n):(g(t)&&(i+="<h5>"+t+" Node</h5>"),""!==e.parallelism&&(i+="<h5>Parallelism: "+e.parallelism+"</h5>"),void 0!==e.lowWatermark&&(i+="<h5>Low Watermark: "+e.lowWatermark+"</h5>"),void 0!==e.operator&&e.operator_strategy&&(i+="<h5>Operation: "+y(e.operator_strategy)+"</h5>")),i+="</div>"},f=function(e,t,r){var n,i;return i="svg-"+e,n="<svg class='"+i+"' width="+t+" height="+r+"><g /></svg>"},y=function(e){var t;for("<"===e.charAt(0)&&(e=e.replace("<","&lt;"),e=e.replace(">","&gt;")),t="";e.length>30;)t=t+e.substring(0,30)+"<br>",e=e.substring(30,e.length);return t+=e},s=function(e,t,r,n,i,o){return nu
 ll==n&&(n=!1),r.id===t.partial_solution?e.setNode(r.id,{label:a(r,"partialSolution",i,o),labelType:"html","class":m(r,"partialSolution")}):r.id===t.next_partial_solution?e.setNode(r.id,{label:a(r,"nextPartialSolution",i,o),labelType:"html","class":m(r,"nextPartialSolution")}):r.id===t.workset?e.setNode(r.id,{label:a(r,"workset",i,o),labelType:"html","class":m(r,"workset")}):r.id===t.next_workset?e.setNode(r.id,{label:a(r,"nextWorkset",i,o),labelType:"html","class":m(r,"nextWorkset")}):r.id===t.solution_set?e.setNode(r.id,{label:a(r,"solutionSet",i,o),labelType:"html","class":m(r,"solutionSet")}):r.id===t.solution_delta?e.setNode(r.id,{label:a(r,"solutionDelta",i,o),labelType:"html","class":m(r,"solutionDelta")}):e.setNode(r.id,{label:a(r,"",i,o),labelType:"html","class":m(r,"")})},i=function(e,t,r,n,i){return e.setEdge(i.id,r.id,{label:o(i),labelType:"html",arrowhead:"normal"})},k=function(e,t){var r,n,o,a,l,u,d,f,p,m,g,h,b,v;for(n=[],null!=t.nodes?v=t.nodes:(v=t.step_function,o=!0)
 ,a=0,u=v.length;a<u;a++)if(r=v[a],p=0,f=0,r.step_function&&(b=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:20,edgesep:0,ranksep:20,rankdir:"LR",marginx:10,marginy:10}),J[r.id]=b,k(b,r),g=new dagreD3.render,c.select("g").call(g,b),p=b.graph().width,f=b.graph().height,angular.element(w).empty()),s(e,t,r,o,p,f),n.push(r.id),null!=r.inputs)for(h=r.inputs,l=0,d=h.length;l<d;l++)m=h[l],i(e,t,r,n,m);return e},M=function(e,t){var r,n,i;for(n in e.nodes){if(r=e.nodes[n],r.id===t)return r;if(null!=r.step_function)for(i in r.step_function)if(r.step_function[i].id===t)return r.step_function[i]}},$=function(e,t){var r,n,i,o;if(!_.isEmpty(t))for(o=e.nodes,r=0,n=o.length;r<n;r++)i=o[r],t[i.id]&&!isNaN(t[i.id].lowWatermark)&&(i.lowWatermark=t[i.id].lowWatermark);return e},b=0,v=0,d=function(){var t,r,n,i,o,a;if(e.plan){p=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:70,edgesep:0,ranksep:50,rankdir:"LR",marginx:40,marginy:40}),k(p,$(e.plan,e.w
 atermarks)),u.selectAll("*").remove(),u.attr("transform","scale(1)"),n=new dagreD3.render,u.call(n,p);for(t in J)i=J[t],l.select("svg.svg-"+t+" g").call(n,i);return r=.5,o=Math.floor((angular.element(S).width()-p.graph().width*r)/2),a=Math.floor((angular.element(S).height()-p.graph().height*r)/2),0!==v&&0!==b?(C.scale(v).translate(b),u.attr("transform","translate("+b+") scale("+v+")")):(C.scale(r).translate([o,a]),u.attr("transform","translate("+o+", "+a+") scale("+C.scale()+")")),C.on("zoom",function(){var e;return e=d3.event,v=e.scale,b=e.translate,u.attr("transform","translate("+b+") scale("+v+")")}),C(l),u.selectAll(".node").on("click",function(t){return e.setNode({nodeid:t})})}},e.$watch(r.plan,function(e){if(e)return d()}),e.$watch(r.watermarks,function(t){if(t&&e.plan)return d()})}}}]),angular.module("flinkApp").service("JobsService",["$http","flinkConfig","$log","amMoment","$q","$timeout",function(e,t,r,n,i,o){var a,s,l,u,c,d;return a=null,s=null,l={},c={running:[],finished:
 [],cancelled:[],failed:[]},u=[],d=function(){return angular.forEach(u,function(e){return e()})},this.registerObserver=function(e){return u.push(e)},this.unRegisterObserver=function(e){var t;return t=u.indexOf(e),u.splice(t,1)},this.stateList=function(){return["SCHEDULED","DEPLOYING","RUNNING","FINISHED","FAILED","CANCELING","CANCELED"]},this.translateLabelState=function(e){switch(e.toLowerCase()){case"finished":return"success";case"failed":return"danger";case"scheduled":return"default";case"deploying":return"info";case"running":return"primary";case"canceling":return"warning";case"pending":return"info";case"total":return"black";default:return"default"}},this.setEndTimes=function(e){return angular.forEach(e,function(e,t){if(!(e["end-time"]>-1))return e["end-time"]=e["start-time"]+e.duration})},this.processVertices=function(e){return angular.forEach(e.vertices,function(e,t){return e.type="regular"}),e.vertices.unshift({name:"Scheduled","start-time":e.timestamps.CREATED,"end-time":e.tim
 estamps.CREATED+1,type:"scheduled"})},this.listJobs=function(){var r;return r=i.defer(),e.get(t.jobServer+"jobs/overview").success(function(e){return function(t,n,i,o){return c.finished=[],c.running=[],_(t.jobs).groupBy(function(e){switch(e.state.toLowerCase()){case"finished":return"finished";case"failed":return"finished";case"canceled":return"finished";default:return"running"}}).forEach(function(t,r){switch(r){case"finished":return c.finished=e.setEndTimes(t);case"running":return c.running=e.setEndTimes(t)}}).value(),r.resolve(c),d()}}(this)),r.promise},this.getJobs=function(e){return c[e]},this.getAllJobs=function(){return c},this.loadJob=function(r){return a=null,l.job=i.defer(),e.get(t.jobServer+"jobs/"+r).success(function(n){return function(i,o,s,u){return n.setEndTimes(i.vertices),n.processVertices(i),e.get(t.jobServer+"jobs/"+r+"/config").success(function(e){return i=angular.extend(i,e),a=i,l.job.resolve(a)})}}(this)),l.job.promise},this.getNode=function(e){var t,r;return r=f
 unction(e,t){var n,i,o,a;for(n=0,i=t.length;n<i;n++){if(o=t[n],o.id===e)return o;if(o.step_function&&(a=r(e,o.step_function)),a)return a}return null},t=i.defer(),l.job.promise.then(function(n){return function(i){var o;return o=r(e,a.plan.nodes),o.vertex=n.seekVertex(e),t.resolve(o)}}(this)),t.promise},this.seekVertex=function(e){var t,r,n,i;for(n=a.vertices,t=0,r=n.length;t<r;t++)if(i=n[t],i.id===e)return i;return null},this.getVertex=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(o){var s;return s=i.seekVertex(r),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/subtasktimes").success(function(e){return s.subtasks=e.subtasks,n.resolve(s)})}}(this)),n.promise},this.getSubtasks=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r).success(function(e){var t;return t=e.subtasks,n.resolve(t)})}}(this)),n.promise},this.getTaskManagers=function(r){var n;return n=i.de
 fer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/taskmanagers").success(function(e){var t;return t=e.taskmanagers,n.resolve(t)})}}(this)),n.promise},this.getAccumulators=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return console.log(a.jid),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/accumulators").success(function(i){var o;return o=i["user-accumulators"],e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/subtasks/accumulators").success(function(e){var t;return t=e.subtasks,n.resolve({main:o,subtasks:t})})})}}(this)),n.promise},this.getCheckpointConfig=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/config").success(function(e){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointStats=function(){var r;return r=i.defer(),l.job.promise.then(func
 tion(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints").success(function(e,t,n,i){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointDetails=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/details/"+r).success(function(e){return angular.equals({},e)?n.resolve(null):n.resolve(e)})}}(this)),n.promise},this.getCheckpointSubtaskDetails=function(r,n){var o;return o=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/details/"+r+"/subtasks/"+n).success(function(e){return angular.equals({},e)?o.resolve(null):o.resolve(e)})}}(this)),o.promise},this.getOperatorBackPressure=function(r){var n;return n=i.defer(),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/backpressure").success(function(e){return function(e){return n.resolve(e)}}(this)),n.promise},this.translateBackPre
 ssureLabelState=function(e){switch(e.toLowerCase()){case"in-progress":return"danger";case"ok":return"success";case"low":return"warning";case"high":return"danger";default:return"default"}},this.loadExceptions=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/exceptions").success(function(e){return a.exceptions=e,r.resolve(e)})}}(this)),r.promise},this.cancelJob=function(r){return e.get(t.jobServer+"jobs/"+r+"/yarn-cancel")},this.stopJob=function(t){return e.get("jobs/"+t+"/yarn-stop")},this}]),angular.module("flinkApp").directive("metricsGraph",function(){return{template:'<div class="panel panel-default panel-metric"> <div class="panel-heading"> <span class="metric-title">{{metric.id}}</span> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.size != \'big\'}]" ng-click="setSize(\'small\')">Small</button> <button type="button" ng-class="[btnClasses, {ac
 tive: metric.size == \'big\'}]" ng-click="setSize(\'big\')">Big</button> </div> <a title="Remove" class="btn btn-default btn-xs remove" ng-click="removeMetric()"><i class="fa fa-close" /></a> </div> </div> <div class="panel-body"> <svg ng-if="metric.view == \'chart\'"/> <div ng-if="metric.view != \'chart\'"> <div class="metric-numeric" title="{{value | humanizeChartNumericTitle:metric}}">{{value | humanizeChartNumeric:metric}}</div> </div> </div> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.view == \'chart\'}]" ng-click="setView(\'chart\')">Chart</button> <button type="button" ng-class="[btnClasses, {active: metric.view != \'chart\'}]" ng-click="setView(\'numeric\')">Numeric</button> </div> </div>',replace:!0,scope:{metric:"=",window:"=",removeMetric:"&",setMetricSize:"=",setMetricView:"=",getValues:"&"},link:function(e,t,r){return e.btnClasses=["btn","btn-default","btn-xs"],e.value=null,e.data=[{values:e.getValues()}],e.
 options={x:function(e,t){return e.x},y:function(e,t){return e.y},xTickFormat:function(e){return d3.time.format("%H:%M:%S")(new Date(e));
-},yTickFormat:function(e){var t,r,n,i;for(r=!1,n=0,i=1,t=Math.abs(e);!r&&n<50;)Math.pow(10,n)<=t&&t<Math.pow(10,n+i)?r=!0:n+=i;return r&&n>6?e/Math.pow(10,n)+"E"+n:""+e}},e.showChart=function(){return d3.select(t.find("svg")[0]).datum(e.data).transition().duration(250).call(e.chart)},e.chart=nv.models.lineChart().options(e.options).showLegend(!1).margin({top:15,left:60,bottom:30,right:30}),e.chart.yAxis.showMaxMin(!1),e.chart.tooltip.hideDelay(0),e.chart.tooltip.contentGenerator(function(e){return"<p>"+d3.time.format("%H:%M:%S")(new Date(e.point.x))+" | "+e.point.y+"</p>"}),nv.utils.windowResize(e.chart.update),e.setSize=function(t){return e.setMetricSize(e.metric,t)},e.setView=function(t){if(e.setMetricView(e.metric,t),"chart"===t)return e.showChart()},"chart"===e.metric.view&&e.showChart(),e.$on("metrics:data:update",function(t,r,n){return e.value=parseFloat(n[e.metric.id]),e.data[0].values.push({x:r,y:e.value}),e.data[0].values.length>e.window&&e.data[0].values.shift(),"chart"===
 e.metric.view&&e.showChart(),"chart"===e.metric.view&&e.chart.clearHighlights(),e.chart.tooltip.hidden(!0)}),t.find(".metric-title").qtip({content:{text:e.metric.id},position:{my:"bottom left",at:"top left"},style:{classes:"qtip-light qtip-timeline-bar"}})}}}),angular.module("flinkApp").service("MetricsService",["$http","$q","flinkConfig","$interval",function(e,t,r,n){return this.metrics={},this.values={},this.watched={},this.observer={jobid:null,nodeid:null,callback:null},this.refresh=n(function(e){return function(){return angular.forEach(e.metrics,function(t,r){return angular.forEach(t,function(t,n){var i;if(i=[],angular.forEach(t,function(e,t){return i.push(e.id)}),i.length>0)return e.getMetrics(r,n,i).then(function(t){if(r===e.observer.jobid&&n===e.observer.nodeid&&e.observer.callback)return e.observer.callback(t)})})})}}(this),r["refresh-interval"]),this.registerObserver=function(e,t,r){return this.observer.jobid=e,this.observer.nodeid=t,this.observer.callback=r},this.unRegiste
 rObserver=function(){return this.observer={jobid:null,nodeid:null,callback:null}},this.setupMetrics=function(e,t){return this.setupLS(),this.watched[e]=[],angular.forEach(t,function(t){return function(r,n){if(r.id)return t.watched[e].push(r.id)}}(this))},this.getWindow=function(){return 100},this.setupLS=function(){return null==sessionStorage.flinkMetrics&&this.saveSetup(),this.metrics=JSON.parse(sessionStorage.flinkMetrics)},this.saveSetup=function(){return sessionStorage.flinkMetrics=JSON.stringify(this.metrics)},this.saveValue=function(e,t,r){if(null==this.values[e]&&(this.values[e]={}),null==this.values[e][t]&&(this.values[e][t]=[]),this.values[e][t].push(r),this.values[e][t].length>this.getWindow())return this.values[e][t].shift()},this.getValues=function(e,t,r){var n;return null==this.values[e]?[]:null==this.values[e][t]?[]:(n=[],angular.forEach(this.values[e][t],function(e){return function(e,t){if(null!=e.values[r])return n.push({x:e.timestamp,y:e.values[r]})}}(this)),n)},thi
 s.setupLSFor=function(e,t){if(null==this.metrics[e]&&(this.metrics[e]={}),null==this.metrics[e][t])return this.metrics[e][t]=[]},this.addMetric=function(e,t,r){return this.setupLSFor(e,t),this.metrics[e][t].push({id:r,size:"small",view:"chart"}),this.saveSetup()},this.removeMetric=function(e){return function(t,r,n){var i;if(null!=e.metrics[t][r])return i=e.metrics[t][r].indexOf(n),i===-1&&(i=_.findIndex(e.metrics[t][r],{id:n})),i!==-1&&e.metrics[t][r].splice(i,1),e.saveSetup()}}(this),this.setMetricSize=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:i,view:n.view}),e.saveSetup()}}(this),this.setMetricView=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:n.size,view:i}),e.saveSetup()}}
 (this),this.orderMetrics=function(e,t,r,n){return this.setupLSFor(e,t),angular.forEach(this.metrics[e][t],function(i){return function(o,a){if(o.id===r.id&&(i.metrics[e][t].splice(a,1),a<n))return n-=1}}(this)),this.metrics[e][t].splice(n,0,r),this.saveSetup()},this.getMetricsSetup=function(e){return function(t,r){return{names:_.map(e.metrics[t][r],function(e){return _.isString(e)?{id:e,size:"small",view:"chart"}:e})}}}(this),this.getAvailableMetrics=function(n){return function(i,o){var a;return n.setupLSFor(i,o),a=t.defer(),e.get(r.jobServer+"jobs/"+i+"/vertices/"+o+"/metrics").success(function(e){var t;return t=[],angular.forEach(e,function(e,r){var a;if(a=n.metrics[i][o].indexOf(e.id),a===-1&&(a=_.findIndex(n.metrics[i][o],{id:e.id})),a===-1)return t.push(e)}),a.resolve(t)}),a.promise}}(this),this.getAllAvailableMetrics=function(n){return function(n,i){var o;return o=t.defer(),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics").success(function(e){return o.resolve(e)}),o.promis
 e}}(this),this.getMetrics=function(n,i,o){var a,s;return a=t.defer(),s=o.join(","),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics?get="+s).success(function(e){return function(t){var r,o;return o={},angular.forEach(t,function(e,t){return o[e.id]=parseInt(e.value)}),r={timestamp:Date.now(),values:o},e.saveValue(n,i,r),a.resolve(r)}}(this)),a.promise},this.setupLS(),this}]),angular.module("flinkApp").controller("JobManagerConfigController",["$scope","JobManagerConfigService",function(e,t){return t.loadConfig().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.config=t})}]).controller("JobManagerLogsController",["$scope","JobManagerLogsService",function(e,t){return t.loadLogs().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.log=t}),e.reloadData=function(){return t.loadLogs().then(function(t){return e.jobmanager.log=t})}}]).controller("JobManagerStdoutController",["$scope","JobManagerStdoutService",function(e,t){return t.loadStdo
 ut().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.stdout=t}),e.reloadData=function(){return t.loadStdout().then(function(t){return e.jobmanager.stdout=t})}}]),angular.module("flinkApp").service("JobManagerConfigService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"jobmanager/config").success(function(e,t,r,i){return i=e,n.resolve(e)}),n.promise},this}]).service("JobManagerLogsService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadLogs=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/log").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]).service("JobManagerStdoutService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadStdout=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/stdout").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module
 ("flinkApp").controller("OverviewController",["$scope","OverviewService","JobsService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.jobObserver=function(){return e.runningJobs=r.getJobs("running"),e.finishedJobs=r.getJobs("finished")},r.registerObserver(e.jobObserver),e.$on("$destroy",function(){return r.unRegisterObserver(e.jobObserver)}),e.jobObserver(),t.loadOverview().then(function(t){return e.overview=t}),o=n(function(){return t.loadOverview().then(function(t){return e.overview=t})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]),angular.module("flinkApp").service("OverviewService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadOverview=function(){var i;return i=r.defer(),e.get(t.jobServer+"overview").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("JobSubmitController",["$scope","JobSubmitService","$interval","flinkConfig","$state","$location",functio
 n(e,t,r,n,i,o){var a;return e.yarn=o.absUrl().indexOf("/proxy/application_")!==-1,e.loadList=function(){return t.loadJarList().then(function(t){return e.address=t.address,e.noaccess=t.error,e.jars=t.files})},e.defaultState=function(){return e.plan=null,e.error=null,e.state={selected:null,parallelism:"",savepointPath:"",allowNonRestoredState:!1,"entry-class":"","program-args":"","plan-button":"Show Plan","submit-button":"Submit","action-time":0}},e.defaultState(),e.uploader={},e.loadList(),a=r(function(){return e.loadList()},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(a)}),e.selectJar=function(t){return e.state.selected===t?e.defaultState():(e.defaultState(),e.state.selected=t)},e.deleteJar=function(r,n){return e.state.selected===n&&e.defaultState(),angular.element(r.currentTarget).removeClass("fa-remove").addClass("fa-spin fa-spinner"),t.deleteJar(n).then(function(e){if(angular.element(r.currentTarget).removeClass("fa-spin fa-spinner").addClass("fa-remove"),nu
 ll!=e.error)return alert(e.error)})},e.loadEntryClass=function(t){return e.state["entry-class"]=t},e.getPlan=function(){var r;if("Show Plan"===e.state["plan-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submit",e.state["plan-button"]="Getting Plan",e.error=null,e.plan=null,t.getPlan(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"]}).then(function(t){if(r===e.state["action-time"])return e.state["plan-button"]="Show Plan",e.error=t.error,e.plan=t.plan})},e.runJob=function(){var r;if("Submit"===e.state["submit-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submitting",e.state["plan-button"]="Show Plan",e.error=null,t.runJob(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"],savepointPath:e.state.savepointPath,allowNonRestoredState:e.state.allowNonRest
 oredState}).then(function(t){if(r===e.state["action-time"]&&(e.state["submit-button"]="Submit",e.error=t.error,null!=t.jobid))return i.go("single-job.plan.subtasks",{jobid:t.jobid})})["catch"](function(t){return e.state["submit-button"]="Submit",e.error=t})},e.nodeid=null,e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.$broadcast("reload")):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null)},e.clearFiles=function(){return e.uploader={}},e.uploadFiles=function(t){return e.uploader={},1===t.length?(e.uploader.file=t[0],e.uploader.upload=!0):e.uploader.error="Did ya forget to select a file?"},e.startUpload=function(){var t,r;return null!=e.uploader.file?(t=new FormData,t.append("jarfile",e.uploader.file),e.uploader.upload=!1,e.uploader.success="Initializing upload...",r=new XMLHttpRequest,r.upload.onprogress=function(t){return e.uploader.success=null,e.uploader.progress=parseInt(100*t.loaded/t
 .total)},r.upload.onerror=function(t){return e.uploader.progress=null,e.uploader.error="An error occurred while uploading your file"},r.upload.onload=function(t){return e.uploader.progress=null,e.uploader.success="Saving..."},r.onreadystatechange=function(){var t;if(4===r.readyState)return t=JSON.parse(r.responseText),null!=t.error?(e.uploader.error=t.error,e.uploader.success=null):e.uploader.success="Uploaded!"},r.open("POST",n.jobServer+"jars/upload"),r.send(t)):console.log("Unexpected Error. This should not happen")}}]).filter("getJarSelectClass",function(){return function(e,t){return e===t?"fa-check-square":"fa-square-o"}}),angular.module("flinkApp").service("JobSubmitService",["$http","flinkConfig","$q",function(e,t,r){return this.loadJarList=function(){var n;return n=r.defer(),e.get(t.jobServer+"jars/").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this.deleteJar=function(n){var i;return i=r.defer(),e["delete"](t.jobServer+"jars/"+encodeURIComponent(n)).success(fu
 nction(e,t,r,n){return i.resolve(e)}),i.promise},this.getPlan=function(n,i){var o;return o=r.defer(),e.get(t.jobServer+"jars/"+encodeURIComponent(n)+"/plan",{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this.runJob=function(n,i){var o;return o=r.defer(),e.post(t.jobServer+"jars/"+encodeURIComponent(n)+"/run",{},{params:i}).success(function(e,t,r,n){return o.resolve(e)}).error(function(e){return o.reject(e)}),o.promise},this}]),angular.module("flinkApp").controller("AllTaskManagersController",["$scope","TaskManagersService","$interval","flinkConfig",function(e,t,r,n){var i;return t.loadManagers().then(function(t){return e.managers=t}),i=r(function(){return t.loadManagers().then(function(t){return e.managers=t})},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(i)})}]).controller("SingleTaskManagerController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.metrics={},r.loadMetri
 cs(t.taskmanagerid).then(function(t){return e.metrics=t}),o=n(function(){return r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]).controller("SingleTaskManagerLogsController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.log={},e.taskmanagerid=t.taskmanagerid,r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t}),e.reloadData=function(){return r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t})}}]).controller("SingleTaskManagerStdoutController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.stdout={},e.taskmanagerid=t.taskmanagerid,r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t}),e.reloadData=function(){return r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t})}}]),angular.module("flinkApp").service("TaskManagersService",
 ["$http","flinkConfig","$q",function(e,t,r){return this.loadManagers=function(){var n;return n=r.defer(),e.get(t.jobServer+"taskmanagers").success(function(e,t,r,i){return n.resolve(e.taskmanagers)}),n.promise},this}]).service("SingleTaskManagerService",["$http","flinkConfig","$q",function(e,t,r){return this.loadMetrics=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n).success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.loadLogs=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/log").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.loadStdout=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/stdout").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this}]);
\ No newline at end of file
+angular.module("flinkApp",["ui.router","angularMoment","dndLists"]).run(["$rootScope",function(e){return e.sidebarVisible=!1,e.showSidebar=function(){return e.sidebarVisible=!e.sidebarVisible,e.sidebarClass="force-show"}}]).value("flinkConfig",{jobServer:"","refresh-interval":1e4}).value("watermarksConfig",{noWatermark:-0x8000000000000000}).run(["JobsService","MainService","flinkConfig","$interval",function(e,t,r,n){return t.loadConfig().then(function(t){return angular.extend(r,t),e.listJobs(),n(function(){return e.listJobs()},r["refresh-interval"])})}]).config(["$uiViewScrollProvider",function(e){return e.useAnchorScroll()}]).run(["$rootScope","$state",function(e,t){return e.$on("$stateChangeStart",function(e,r,n,i){if(r.redirectTo)return e.preventDefault(),t.go(r.redirectTo,n)})}]).config(["$stateProvider","$urlRouterProvider",function(e,t){return e.state("overview",{url:"/overview",views:{main:{templateUrl:"partials/overview.html",controller:"OverviewController"}}}).state("runnin
 g-jobs",{url:"/running-jobs",views:{main:{templateUrl:"partials/jobs/running-jobs.html",controller:"RunningJobsController"}}}).state("completed-jobs",{url:"/completed-jobs",views:{main:{templateUrl:"partials/jobs/completed-jobs.html",controller:"CompletedJobsController"}}}).state("single-job",{url:"/jobs/{jobid}","abstract":!0,views:{main:{templateUrl:"partials/jobs/job.html",controller:"SingleJobController"}}}).state("single-job.plan",{url:"",redirectTo:"single-job.plan.subtasks",views:{details:{templateUrl:"partials/jobs/job.plan.html",controller:"JobPlanController"}}}).state("single-job.plan.subtasks",{url:"",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.subtasks.html",controller:"JobPlanSubtasksController"}}}).state("single-job.plan.metrics",{url:"/metrics",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.metrics.html",controller:"JobPlanMetricsController"}}}).state("single-job.plan.watermarks",{url:"/watermarks",views:{"node-details":{
 templateUrl:"partials/jobs/job.plan.node-list.watermarks.html"}}}).state("single-job.plan.accumulators",{url:"/accumulators",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.accumulators.html",controller:"JobPlanAccumulatorsController"}}}).state("single-job.plan.checkpoints",{url:"/checkpoints",redirectTo:"single-job.plan.checkpoints.overview",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.checkpoints.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.overview",{url:"/overview",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.overview.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.summary",{url:"/summary",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.summary.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.history",{url:"/history",views:{"checkpoints-view":{template
 Url:"partials/jobs/job.plan.node.checkpoints.history.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.config",{url:"/config",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.config.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.details",{url:"/details/{checkpointId}",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.details.html",controller:"JobPlanCheckpointDetailsController"}}}).state("single-job.plan.backpressure",{url:"/backpressure",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.backpressure.html",controller:"JobPlanBackPressureController"}}}).state("single-job.timeline",{url:"/timeline",views:{details:{templateUrl:"partials/jobs/job.timeline.html"}}}).state("single-job.timeline.vertex",{url:"/{vertexId}",views:{vertex:{templateUrl:"partials/jobs/job.timeline.vertex.html",controller:"JobTimelineVertexController"}}}).sta
 te("single-job.exceptions",{url:"/exceptions",views:{details:{templateUrl:"partials/jobs/job.exceptions.html",controller:"JobExceptionsController"}}}).state("single-job.config",{url:"/config",views:{details:{templateUrl:"partials/jobs/job.config.html"}}}).state("all-manager",{url:"/taskmanagers",views:{main:{templateUrl:"partials/taskmanager/index.html",controller:"AllTaskManagersController"}}}).state("single-manager",{url:"/taskmanager/{taskmanagerid}","abstract":!0,views:{main:{templateUrl:"partials/taskmanager/taskmanager.html",controller:"SingleTaskManagerController"}}}).state("single-manager.metrics",{url:"/metrics",views:{details:{templateUrl:"partials/taskmanager/taskmanager.metrics.html"}}}).state("single-manager.stdout",{url:"/stdout",views:{details:{templateUrl:"partials/taskmanager/taskmanager.stdout.html",controller:"SingleTaskManagerStdoutController"}}}).state("single-manager.log",{url:"/log",views:{details:{templateUrl:"partials/taskmanager/taskmanager.log.html",contro
 ller:"SingleTaskManagerLogsController"}}}).state("jobmanager",{url:"/jobmanager",views:{main:{templateUrl:"partials/jobmanager/index.html"}}}).state("jobmanager.config",{url:"/config",views:{details:{templateUrl:"partials/jobmanager/config.html",controller:"JobManagerConfigController"}}}).state("jobmanager.stdout",{url:"/stdout",views:{details:{templateUrl:"partials/jobmanager/stdout.html",controller:"JobManagerStdoutController"}}}).state("jobmanager.log",{url:"/log",views:{details:{templateUrl:"partials/jobmanager/log.html",controller:"JobManagerLogsController"}}}).state("submit",{url:"/submit",views:{main:{templateUrl:"partials/submit.html",controller:"JobSubmitController"}}}),t.otherwise("/overview")}]),angular.module("flinkApp").directive("bsLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getLab
 elClass=function(){return"label label-"+e.translateLabelState(n.status)}}}}]).directive("bpLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getBackPressureLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getBackPressureLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getBackPressureLabelClass=function(){return"label label-"+e.translateBackPressureLabelState(n.status)}}}}]).directive("indicatorPrimary",["JobsService",function(e){return{replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<i title='{{status}}' ng-class='getLabelClass()' />",link:function(t,r,n){return t.getLabelClass=function(){return"fa fa-circle indicator indicator-"+e.translateLabelState(n.status)}}}}]).directive("tableProperty",function(){return{replace:!0,scope:{value:"="},template:"<td title=\"{{value || 'None'}}\">{{value || 'None'}}</td>"}}),angular.module("flinkApp").filter("amDurationFormatExtended",["angularMomentConfig",
 function(e){var t;return t=function(e,t,r){return"undefined"==typeof e||null===e?"":moment.duration(e,t).format(r,{trim:!1})},t.$stateful=e.statefulFilters,t}]).filter("humanizeDuration",function(){return function(e,t){var r,n,i,o,a,s;return"undefined"==typeof e||null===e?"":(o=e%1e3,s=Math.floor(e/1e3),a=s%60,s=Math.floor(s/60),i=s%60,s=Math.floor(s/60),n=s%24,s=Math.floor(s/24),r=s,0===r?0===n?0===i?0===a?o+"ms":a+"s ":i+"m "+a+"s":t?n+"h "+i+"m":n+"h "+i+"m "+a+"s":t?r+"d "+n+"h":r+"d "+n+"h "+i+"m "+a+"s")}}).filter("limit",function(){return function(e){return e.length>73&&(e=e.substring(0,35)+"..."+e.substring(e.length-35,e.length)),e}}).filter("humanizeText",function(){return function(e){return e?e.replace(/&gt;/g,">").replace(/<br\/>/g,""):""}}).filter("humanizeBytes",function(){return function(e){var t,r;return r=["B","KB","MB","GB","TB","PB","EB"],t=function(e,n){var i;return i=Math.pow(1024,n),e<i?(e/i).toFixed(2)+" "+r[n]:e<1e3*i?(e/i).toPrecision(3)+" "+r[n]:t(e,n+1)},"u
 ndefined"==typeof e||null===e?"":e<1e3?e+" B":t(e,1)}}).filter("toLocaleString",function(){return function(e){return e.toLocaleString()}}).filter("toUpperCase",function(){return function(e){return e.toUpperCase()}}).filter("percentage",function(){return function(e){return(100*e).toFixed(0)+"%"}}).filter("humanizeWatermark",["watermarksConfig",function(e){return function(t){return isNaN(t)||t<=e.noWatermark?"No Watermark":t}}]).filter("increment",function(){return function(e){return parseInt(e)+1}}).filter("humanizeChartNumeric",["humanizeBytesFilter","humanizeDurationFilter",function(e,t){return function(r,n){var i;return i="",null!==r&&(i=/bytes/i.test(n.id)&&/persecond/i.test(n.id)?e(r)+" / s":/bytes/i.test(n.id)?e(r):/persecond/i.test(n.id)?r+" / s":/time/i.test(n.id)||/latency/i.test(n.id)?t(r,!0):r),i}}]).filter("humanizeChartNumericTitle",["humanizeDurationFilter",function(e){return function(t,r){var n;return n="",null!==t&&(n=/bytes/i.test(r.id)&&/persecond/i.test(r.id)?t+" B
 ytes / s":/bytes/i.test(r.id)?t+" Bytes":/persecond/i.test(r.id)?t+" / s":/time/i.test(r.id)||/latency/i.test(r.id)?e(t,!1):t),n}}]).filter("searchMetrics",function(){return function(e,t){var r,n;return n=new RegExp(t,"gi"),function(){var t,i,o;for(o=[],t=0,i=e.length;t<i;t++)r=e[t],r.id.match(n)&&o.push(r);return o}()}}),angular.module("flinkApp").service("MainService",["$http","flinkConfig","$q",function(e,t,r){return this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"config").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this}]),angular.module("flinkApp").controller("JobManagerConfigController",["$scope","JobManagerConfigService",function(e,t){return t.loadConfig().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.config=t})}]).controller("JobManagerLogsController",["$scope","JobManagerLogsService",function(e,t){return t.loadLogs().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.log=t}),e.reloadD
 ata=function(){return t.loadLogs().then(function(t){return e.jobmanager.log=t})}}]).controller("JobManagerStdoutController",["$scope","JobManagerStdoutService",function(e,t){return t.loadStdout().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.stdout=t}),e.reloadData=function(){return t.loadStdout().then(function(t){return e.jobmanager.stdout=t})}}]),angular.module("flinkApp").service("JobManagerConfigService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"jobmanager/config").success(function(e,t,r,i){return i=e,n.resolve(e)}),n.promise},this}]).service("JobManagerLogsService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadLogs=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/log").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]).service("JobManagerStdoutService",["$http","flinkConfig","$q",function(e,t,r){var
  n;return n={},this.loadStdout=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/stdout").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("RunningJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("running")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("CompletedJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("finished")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("SingleJobController",["$scope","$state","$stateParams","JobsService","MetricsService","$rootScope","flinkConfig","$interval","$q","watermarksConfig",function(e,t,r,n,i,o,a,s,l,u){var c,d;r
 eturn e.jobid=r.jobid,e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats={},d=s(function(){return n.loadJob(r.jobid).then(function(t){return e.job=t,e.$broadcast("reload")})},a["refresh-interval"]),e.$on("$destroy",function(){return e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats=null,s.cancel(d)}),e.cancelJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Cancelling..."),n.cancelJob(r.jobid).then(function(e){return{}})},e.stopJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Stopping..."),n.stopJob(r.jobid).then(function(e){return{}})},n.loadJob(r.jobid).then(function(t){return e.job=t,e.vertices=t.vertices,e.plan=t.plan,i.setupMetrics(r.jobid,t.vertices)}),c=function(t){var r,n,o,a;return o=function(t){return function(t){var r,n,o,a;return r=l.defer(),o=e.job.jid,a=function(){var e,r,i;for(i=[],n=e=0,r
 =t.parallelism-1;0<=r?e<=r:e>=r;n=0<=r?++e:--e)i.push(n+".currentLowWatermark");return i}(),i.getMetrics(o,t.id,a).then(function(e){var t,n,i,o,a,s,l;i=NaN,l={},o=e.values;for(t in o)s=o[t],a=t.replace(".currentLowWatermark",""),l[a]=s,(isNaN(i)||s<i)&&(i=s);return n=!isNaN(i)&&i>u.noWatermark?i:NaN,r.resolve({lowWatermark:n,watermarks:l})}),r.promise}}(this),r=l.defer(),a={},n=t.length,angular.forEach(t,function(e){return function(e,t){var i;return i=e.id,o(e).then(function(e){if(a[i]=e,t>=n-1)return r.resolve(a)})}}(this)),r.promise},e.hasWatermark=function(t){return e.watermarks[t]&&!isNaN(e.watermarks[t].lowWatermark)},e.$watch("plan",function(t){if(t)return c(t.nodes).then(function(t){return e.watermarks=t})}),e.$on("reload",function(){if(e.plan)return c(e.plan.nodes).then(function(t){return e.watermarks=t})})}]).controller("JobPlanController",["$scope","$state","$stateParams","$window","JobsService",function(e,t,r,n,i){return e.nodeid=null,e.nodeUnfolded=!1,e.stateList=i.state
 List(),e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null,e.$broadcast("reload"),e.$broadcast("node:change",e.nodeid)):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null)},e.deactivateNode=function(){return e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null},e.toggleFold=function(){return e.nodeUnfolded=!e.nodeUnfolded}}]).controller("JobPlanSubtasksController",["$scope","JobsService",function(e,t){var r;return e.aggregate=!1,r=function(){return e.aggregate?t.getTaskManagers(e.nodeid).then(function(t){return e.taskmanagers=t}):t.getSubtasks(e.nodeid).then(function(t){return e.subtasks=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanAccumulatorsController",["$scope","JobsService",function(e,t){var r;return r
 =function(){return t.getAccumulators(e.nodeid).then(function(t){return e.accumulators=t.main,e.subtaskAccumulators=t.subtasks})},!e.nodeid||e.vertex&&e.vertex.accumulators||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanCheckpointsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return e.checkpointDetails={},e.checkpointDetails.id=-1,n.getCheckpointConfig().then(function(t){return e.checkpointConfig=t}),i=function(){return n.getCheckpointStats().then(function(t){if(null!==t)return e.checkpointStats=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobPlanCheckpointDetailsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i,o;return e.subtaskDetails={},e.checkpointDetails.id=r.checkpointId,i=function(t){return n.getCheckpointDetails(t).then(function(t){return null!==t?e.checkpoint=t:e.unknown_checkpoint=!0})},o=function(t,r){return n.getCheckpointSubtaskDetails(t,r).then(fun
 ction(t){if(null!==t)return e.subtaskDetails[r]=t})},i(r.checkpointId),e.nodeid&&o(r.checkpointId,e.nodeid),e.$on("reload",function(t){if(i(r.checkpointId),e.nodeid)return o(r.checkpointId,e.nodeid)}),e.$on("$destroy",function(){return e.checkpointDetails.id=-1})}]).controller("JobPlanBackPressureController",["$scope","JobsService",function(e,t){var r;return r=function(){if(e.now=Date.now(),e.nodeid)return t.getOperatorBackPressure(e.nodeid).then(function(t){return e.backPressureOperatorStats[e.nodeid]=t})},r(),e.$on("reload",function(e){return r()})}]).controller("JobTimelineVertexController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return i=function(){return n.getVertex(r.vertexId).then(function(t){return e.vertex=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobExceptionsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return n.loadExceptions().then(function(t){return e.exceptions=t})}]).controller("Job
 PropertiesController",["$scope","JobsService",function(e,t){return e.changeNode=function(r){return r!==e.nodeid?(e.nodeid=r,t.getNode(r).then(function(t){return e.node=t})):(e.nodeid=null,e.node=null)}}]).controller("JobPlanMetricsController",["$scope","JobsService","MetricsService",function(e,t,r){var n,i;if(e.dragging=!1,e.window=r.getWindow(),e.availableMetrics=null,e.$on("$destroy",function(){return r.unRegisterObserver()}),i=function(){return t.getVertex(e.nodeid).then(function(t){return e.vertex=t}),r.getAvailableMetrics(e.jobid,e.nodeid).then(function(t){return e.availableMetrics=t.sort(n),e.metrics=r.getMetricsSetup(e.jobid,e.nodeid).names,r.registerObserver(e.jobid,e.nodeid,function(t){return e.$broadcast("metrics:data:update",t.timestamp,t.values)})})},n=function(e,t){var r,n;return r=e.id.toLowerCase(),n=t.id.toLowerCase(),r<n?-1:r>n?1:0},e.dropped=function(t,n,o,a,s){return r.orderMetrics(e.jobid,e.nodeid,o,n),e.$broadcast("metrics:refresh",o),i(),!1},e.dragStart=functio
 n(){return e.dragging=!0},e.dragEnd=function(){return e.dragging=!1},e.addMetric=function(t){return r.addMetric(e.jobid,e.nodeid,t.id),i()},e.removeMetric=function(t){return r.removeMetric(e.jobid,e.nodeid,t),i()},e.setMetricSize=function(t,n){return r.setMetricSize(e.jobid,e.nodeid,t,n),i()},e.setMetricView=function(t,n){return r.setMetricView(e.jobid,e.nodeid,t,n),i()},e.getValues=function(t){return r.getValues(e.jobid,e.nodeid,t)},e.$on("node:change",function(t,r){if(!e.dragging)return i()}),e.nodeid)return i()}]),angular.module("flinkApp").directive("vertex",["$state",function(e){return{template:"<svg class='timeline secondary' width='0' height='0'></svg>",scope:{data:"="},link:function(e,t,r){var n,i,o;o=t.children()[0],i=t.width(),angular.element(o).attr("width",i),(n=function(e){var t,r,n;return d3.select(o).selectAll("*").remove(),n=[],angular.forEach(e.subtasks,function(e,t){var r;return r=[{label:"Scheduled",color:"#666",borderColor:"#555",starting_time:e.timestamps.SCHEDU
 LED,ending_time:e.timestamps.DEPLOYING,type:"regular"},{label:"Deploying",color:"#aaa",borderColor:"#555",starting_time:e.timestamps.DEPLOYING,ending_time:e.timestamps.RUNNING,type:"regular"}],e.timestamps.FINISHED>0&&r.push({label:"Running",color:"#ddd",borderColor:"#555",starting_time:e.timestamps.RUNNING,ending_time:e.timestamps.FINISHED,type:"regular"}),n.push({label:"("+e.subtask+") "+e.host,times:r})}),t=d3.timeline().stack().tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("single").labelFormat(function(e){return e}).margin({left:100,right:0,top:0,bottom:0}).itemHeight(30).relativeTime(),r=d3.select(o).datum(n).call(t)})(e.data)}}}]).directive("timeline",["$state",function(e){return{template:"<svg class='timeline' width='0' height='0'></svg>",scope:{vertices:"=",jobid:"="},link:function(t,r,n){var i,o,a,s;a=r.children()[0],o=r.width(),angular.element(a).attr("width",o),s=function(e){return e.replace("&gt;",">")},i=function(r){var n,i,o;return d3.select(a).selectAll
 ("*").remove(),o=[],angular.forEach(r,function(e){if(e["start-time"]>-1)return"scheduled"===e.type?o.push({times:[{label:s(e.name),color:"#cccccc",borderColor:"#555555",starting_time:e["start-time"],ending_time:e["end-time"],type:e.type}]}):o.push({times:[{label:s(e.name),color:"#d9f1f7",borderColor:"#62cdea",starting_time:e["start-time"],ending_time:e["end-time"],link:e.id,type:e.type}]})}),n=d3.timeline().stack().click(function(r,n,i){if(r.link)return e.go("single-job.timeline.vertex",{jobid:t.jobid,vertexId:r.link})}).tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("main").margin({left:0,right:0,top:0,bottom:0}).itemHeight(30).showBorderLine().showHourTimeline(),i=d3.select(a).datum(o).call(n)},t.$watch(n.vertices,function(e){if(e)return i(e)})}}}]).directive("split",function(){return{compile:function(e,t){return Split(e.children(),{sizes:[50,50],direction:"vertical"})}}}).directive("jobPlan",["$timeout",function(e){return{template:"<svg class='graph'><g /></svg> <svg
  class='tmp' width='1' height='1'><g /></svg> <div class='btn-group zoom-buttons'> <a class='btn btn-default zoom-in' ng-click='zoomIn()'><i class='fa fa-plus' /></a> <a class='btn btn-default zoom-out' ng-click='zoomOut()'><i class='fa fa-minus' /></a> </div>",scope:{plan:"=",watermarks:"=",setNode:"&"},link:function(e,t,r){var n,i,o,a,s,l,u,c,d,f,p,m,g,h,b,v,k,j,S,w,C,$,M,y,J;p=null,C=d3.behavior.zoom(),J=[],h=r.jobid,S=t.children()[0],j=t.children().children()[0],w=t.children()[1],l=d3.select(S),u=d3.select(j),c=d3.select(w),n=t.width(),angular.element(t.children()[0]).width(n),v=0,b=0,e.zoomIn=function(){var e,t,r;if(C.scale()<2.99)return e=C.translate(),t=e[0]*(C.scale()+.1/C.scale()),r=e[1]*(C.scale()+.1/C.scale()),C.scale(C.scale()+.1),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},e.zoomOut=function(){var e,t,r;if(C.scale()>.31)return C.scale(C.scale()-.1),e=C.translate(),t=e[0]*(C.scale()-.1/C.scale()),r=e[1
 ]*(C.scale()-.1/C.scale()),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},o=function(e){var t;return t="",null==e.ship_strategy&&null==e.local_strategy||(t+="<div class='edge-label'>",null!=e.ship_strategy&&(t+=e.ship_strategy),void 0!==e.temp_mode&&(t+=" ("+e.temp_mode+")"),void 0!==e.local_strategy&&(t+=",<br>"+e.local_strategy),t+="</div>"),t},g=function(e){return"partialSolution"===e||"nextPartialSolution"===e||"workset"===e||"nextWorkset"===e||"solutionSet"===e||"solutionDelta"===e},m=function(e,t){return"mirror"===t?"node-mirror":g(t)?"node-iteration":"node-normal"},a=function(e,t,r,n){var i,o;return i="<div href='#/jobs/"+h+"/vertex/"+e.id+"' class='node-label "+m(e,t)+"'>",i+="mirror"===t?"<h3 class='node-name'>Mirror of "+e.operator+"</h3>":"<h3 class='node-name'>"+e.operator+"</h3>",""===e.description?i+="":(o=e.description,o=y(o),i+="<h4 class='step-name'>"+o+"</h4>"),null!=e.step_function?i+=f(e.id,r,n):(
 g(t)&&(i+="<h5>"+t+" Node</h5>"),""!==e.parallelism&&(i+="<h5>Parallelism: "+e.parallelism+"</h5>"),void 0!==e.lowWatermark&&(i+="<h5>Low Watermark: "+e.lowWatermark+"</h5>"),void 0!==e.operator&&e.operator_strategy&&(i+="<h5>Operation: "+y(e.operator_strategy)+"</h5>")),i+="</div>"},f=function(e,t,r){var n,i;return i="svg-"+e,n="<svg class='"+i+"' width="+t+" height="+r+"><g /></svg>"},y=function(e){var t;for("<"===e.charAt(0)&&(e=e.replace("<","&lt;"),e=e.replace(">","&gt;")),t="";e.length>30;)t=t+e.substring(0,30)+"<br>",e=e.substring(30,e.length);return t+=e},s=function(e,t,r,n,i,o){return null==n&&(n=!1),r.id===t.partial_solution?e.setNode(r.id,{label:a(r,"partialSolution",i,o),labelType:"html","class":m(r,"partialSolution")}):r.id===t.next_partial_solution?e.setNode(r.id,{label:a(r,"nextPartialSolution",i,o),labelType:"html","class":m(r,"nextPartialSolution")}):r.id===t.workset?e.setNode(r.id,{label:a(r,"workset",i,o),labelType:"html","class":m(r,"workset")}):r.id===t.next_wor
 kset?e.setNode(r.id,{label:a(r,"nextWorkset",i,o),labelType:"html","class":m(r,"nextWorkset")}):r.id===t.solution_set?e.setNode(r.id,{label:a(r,"solutionSet",i,o),labelType:"html","class":m(r,"solutionSet")}):r.id===t.solution_delta?e.setNode(r.id,{label:a(r,"solutionDelta",i,o),labelType:"html","class":m(r,"solutionDelta")}):e.setNode(r.id,{label:a(r,"",i,o),labelType:"html","class":m(r,"")})},i=function(e,t,r,n,i){return e.setEdge(i.id,r.id,{label:o(i),labelType:"html",arrowhead:"normal"})},k=function(e,t){var r,n,o,a,l,u,d,f,p,m,g,h,b,v;for(n=[],null!=t.nodes?v=t.nodes:(v=t.step_function,o=!0),a=0,u=v.length;a<u;a++)if(r=v[a],p=0,f=0,r.step_function&&(b=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:20,edgesep:0,ranksep:20,rankdir:"LR",marginx:10,marginy:10}),J[r.id]=b,k(b,r),g=new dagreD3.render,c.select("g").call(g,b),p=b.graph().width,f=b.graph().height,angular.element(w).empty()),s(e,t,r,o,p,f),n.push(r.id),null!=r.inputs)for(h=r.inputs,l=0,d=h.leng
 th;l<d;l++)m=h[l],i(e,t,r,n,m);return e},M=function(e,t){var r,n,i;for(n in e.nodes){if(r=e.nodes[n],r.id===t)return r;if(null!=r.step_function)for(i in r.step_function)if(r.step_function[i].id===t)return r.step_function[i]}},$=function(e,t){var r,n,i,o;if(!_.isEmpty(t))for(o=e.nodes,r=0,n=o.length;r<n;r++)i=o[r],t[i.id]&&!isNaN(t[i.id].lowWatermark)&&(i.lowWatermark=t[i.id].lowWatermark);return e},b=0,v=0,d=function(){var t,r,n,i,o,a;if(e.plan){p=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:70,edgesep:0,ranksep:50,rankdir:"LR",marginx:40,marginy:40}),k(p,$(e.plan,e.watermarks)),u.selectAll("*").remove(),u.attr("transform","scale(1)"),n=new dagreD3.render,u.call(n,p);for(t in J)i=J[t],l.select("svg.svg-"+t+" g").call(n,i);return r=.5,o=Math.floor((angular.element(S).width()-p.graph().width*r)/2),a=Math.floor((angular.element(S).height()-p.graph().height*r)/2),0!==v&&0!==b?(C.scale(v).translate(b),u.attr("transform","translate("+b+") scale("+v+")")):(C.sc
 ale(r).translate([o,a]),u.attr("transform","translate("+o+", "+a+") scale("+C.scale()+")")),C.on("zoom",function(){var e;return e=d3.event,v=e.scale,b=e.translate,u.attr("transform","translate("+b+") scale("+v+")")}),C(l),u.selectAll(".node").on("click",function(t){return e.setNode({nodeid:t})})}},e.$watch(r.plan,function(e){if(e)return d()}),e.$watch(r.watermarks,function(t){if(t&&e.plan)return d()})}}}]),angular.module("flinkApp").service("JobsService",["$http","flinkConfig","$log","amMoment","$q","$timeout",function(e,t,r,n,i,o){var a,s,l,u,c,d;return a=null,s=null,l={},c={running:[],finished:[],cancelled:[],failed:[]},u=[],d=function(){return angular.forEach(u,function(e){return e()})},this.registerObserver=function(e){return u.push(e)},this.unRegisterObserver=function(e){var t;return t=u.indexOf(e),u.splice(t,1)},this.stateList=function(){return["SCHEDULED","DEPLOYING","RUNNING","FINISHED","FAILED","CANCELING","CANCELED"]},this.translateLabelState=function(e){switch(e.toLowerCa
 se()){case"finished":return"success";case"failed":return"danger";case"scheduled":return"default";case"deploying":return"info";case"running":return"primary";case"canceling":return"warning";case"pending":return"info";case"total":return"black";default:return"default"}},this.setEndTimes=function(e){return angular.forEach(e,function(e,t){if(!(e["end-time"]>-1))return e["end-time"]=e["start-time"]+e.duration})},this.processVertices=function(e){return angular.forEach(e.vertices,function(e,t){return e.type="regular"}),e.vertices.unshift({name:"Scheduled","start-time":e.timestamps.CREATED,"end-time":e.timestamps.CREATED+1,type:"scheduled"})},this.listJobs=function(){var r;return r=i.defer(),e.get(t.jobServer+"jobs/overview").success(function(e){return function(t,n,i,o){return c.finished=[],c.running=[],_(t.jobs).groupBy(function(e){switch(e.state.toLowerCase()){case"finished":return"finished";case"failed":return"finished";case"canceled":return"finished";default:return"running"}}).forEach(fun
 ction(t,r){switch(r){case"finished":return c.finished=e.setEndTimes(t);case"running":return c.running=e.setEndTimes(t)}}).value(),r.resolve(c),d()}}(this)),r.promise},this.getJobs=function(e){return c[e]},this.getAllJobs=function(){return c},this.loadJob=function(r){return a=null,l.job=i.defer(),e.get(t.jobServer+"jobs/"+r).success(function(n){return function(i,o,s,u){return n.setEndTimes(i.vertices),n.processVertices(i),e.get(t.jobServer+"jobs/"+r+"/config").success(function(e){return i=angular.extend(i,e),a=i,l.job.resolve(a)})}}(this)),l.job.promise},this.getNode=function(e){var t,r;return r=function(e,t){var n,i,o,a;for(n=0,i=t.length;n<i;n++){if(o=t[n],o.id===e)return o;if(o.step_function&&(a=r(e,o.step_function)),a)return a}return null},t=i.defer(),l.job.promise.then(function(n){return function(i){var o;return o=r(e,a.plan.nodes),o.vertex=n.seekVertex(e),t.resolve(o)}}(this)),t.promise},this.seekVertex=function(e){var t,r,n,i;for(n=a.vertices,t=0,r=n.length;t<r;t++)if(i=n[t],i
 .id===e)return i;return null},this.getVertex=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(o){var s;return s=i.seekVertex(r),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/subtasktimes").success(function(e){return s.subtasks=e.subtasks,n.resolve(s)})}}(this)),n.promise},this.getSubtasks=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r).success(function(e){var t;return t=e.subtasks,n.resolve(t)})}}(this)),n.promise},this.getTaskManagers=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/taskmanagers").success(function(e){var t;return t=e.taskmanagers,n.resolve(t)})}}(this)),n.promise},this.getAccumulators=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return console.log(a.jid),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"
 /accumulators").success(function(i){var o;return o=i["user-accumulators"],e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/subtasks/accumulators").success(function(e){var t;return t=e.subtasks,n.resolve({main:o,subtasks:t})})})}}(this)),n.promise},this.getCheckpointConfig=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/config").success(function(e){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointStats=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints").success(function(e,t,n,i){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointDetails=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/details/"+r).success(function(e){return a
 ngular.equals({},e)?n.resolve(null):n.resolve(e)})}}(this)),n.promise},this.getCheckpointSubtaskDetails=function(r,n){var o;return o=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+a.jid+"/checkpoints/details/"+r+"/subtasks/"+n).success(function(e){return angular.equals({},e)?o.resolve(null):o.resolve(e)})}}(this)),o.promise},this.getOperatorBackPressure=function(r){var n;return n=i.defer(),e.get(t.jobServer+"jobs/"+a.jid+"/vertices/"+r+"/backpressure").success(function(e){return function(e){return n.resolve(e)}}(this)),n.promise},this.translateBackPressureLabelState=function(e){switch(e.toLowerCase()){case"in-progress":return"danger";case"ok":return"success";case"low":return"warning";case"high":return"danger";default:return"default"}},this.loadExceptions=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+a.jid+"/exceptions").success(function(e){return a.exceptions=e,r.re
 solve(e)})}}(this)),r.promise},this.cancelJob=function(r){return e.get(t.jobServer+"jobs/"+r+"/yarn-cancel"),
+e.patch(t.jobServer+"jobs/"+r+"?mode=cancel")},this.stopJob=function(r){return e.get("jobs/"+r+"/yarn-stop"),e.patch(t.jobServer+"jobs/"+r+"?mode=stop")},this}]),angular.module("flinkApp").directive("metricsGraph",function(){return{template:'<div class="panel panel-default panel-metric"> <div class="panel-heading"> <span class="metric-title">{{metric.id}}</span> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.size != \'big\'}]" ng-click="setSize(\'small\')">Small</button> <button type="button" ng-class="[btnClasses, {active: metric.size == \'big\'}]" ng-click="setSize(\'big\')">Big</button> </div> <a title="Remove" class="btn btn-default btn-xs remove" ng-click="removeMetric()"><i class="fa fa-close" /></a> </div> </div> <div class="panel-body"> <svg ng-if="metric.view == \'chart\'"/> <div ng-if="metric.view != \'chart\'"> <div class="metric-numeric" title="{{value | humanizeChartNumericTitle:metric}}">{{value | humanizeChar
 tNumeric:metric}}</div> </div> </div> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.view == \'chart\'}]" ng-click="setView(\'chart\')">Chart</button> <button type="button" ng-class="[btnClasses, {active: metric.view != \'chart\'}]" ng-click="setView(\'numeric\')">Numeric</button> </div> </div>',replace:!0,scope:{metric:"=",window:"=",removeMetric:"&",setMetricSize:"=",setMetricView:"=",getValues:"&"},link:function(e,t,r){return e.btnClasses=["btn","btn-default","btn-xs"],e.value=null,e.data=[{values:e.getValues()}],e.options={x:function(e,t){return e.x},y:function(e,t){return e.y},xTickFormat:function(e){return d3.time.format("%H:%M:%S")(new Date(e))},yTickFormat:function(e){var t,r,n,i;for(r=!1,n=0,i=1,t=Math.abs(e);!r&&n<50;)Math.pow(10,n)<=t&&t<Math.pow(10,n+i)?r=!0:n+=i;return r&&n>6?e/Math.pow(10,n)+"E"+n:""+e}},e.showChart=function(){return d3.select(t.find("svg")[0]).datum(e.data).transition().duration(250).call(e.c
 hart)},e.chart=nv.models.lineChart().options(e.options).showLegend(!1).margin({top:15,left:60,bottom:30,right:30}),e.chart.yAxis.showMaxMin(!1),e.chart.tooltip.hideDelay(0),e.chart.tooltip.contentGenerator(function(e){return"<p>"+d3.time.format("%H:%M:%S")(new Date(e.point.x))+" | "+e.point.y+"</p>"}),nv.utils.windowResize(e.chart.update),e.setSize=function(t){return e.setMetricSize(e.metric,t)},e.setView=function(t){if(e.setMetricView(e.metric,t),"chart"===t)return e.showChart()},"chart"===e.metric.view&&e.showChart(),e.$on("metrics:data:update",function(t,r,n){return e.value=parseFloat(n[e.metric.id]),e.data[0].values.push({x:r,y:e.value}),e.data[0].values.length>e.window&&e.data[0].values.shift(),"chart"===e.metric.view&&e.showChart(),"chart"===e.metric.view&&e.chart.clearHighlights(),e.chart.tooltip.hidden(!0)}),t.find(".metric-title").qtip({content:{text:e.metric.id},position:{my:"bottom left",at:"top left"},style:{classes:"qtip-light qtip-timeline-bar"}})}}}),angular.module("f
 linkApp").service("MetricsService",["$http","$q","flinkConfig","$interval",function(e,t,r,n){return this.metrics={},this.values={},this.watched={},this.observer={jobid:null,nodeid:null,callback:null},this.refresh=n(function(e){return function(){return angular.forEach(e.metrics,function(t,r){return angular.forEach(t,function(t,n){var i;if(i=[],angular.forEach(t,function(e,t){return i.push(e.id)}),i.length>0)return e.getMetrics(r,n,i).then(function(t){if(r===e.observer.jobid&&n===e.observer.nodeid&&e.observer.callback)return e.observer.callback(t)})})})}}(this),r["refresh-interval"]),this.registerObserver=function(e,t,r){return this.observer.jobid=e,this.observer.nodeid=t,this.observer.callback=r},this.unRegisterObserver=function(){return this.observer={jobid:null,nodeid:null,callback:null}},this.setupMetrics=function(e,t){return this.setupLS(),this.watched[e]=[],angular.forEach(t,function(t){return function(r,n){if(r.id)return t.watched[e].push(r.id)}}(this))},this.getWindow=function
 (){return 100},this.setupLS=function(){return null==sessionStorage.flinkMetrics&&this.saveSetup(),this.metrics=JSON.parse(sessionStorage.flinkMetrics)},this.saveSetup=function(){return sessionStorage.flinkMetrics=JSON.stringify(this.metrics)},this.saveValue=function(e,t,r){if(null==this.values[e]&&(this.values[e]={}),null==this.values[e][t]&&(this.values[e][t]=[]),this.values[e][t].push(r),this.values[e][t].length>this.getWindow())return this.values[e][t].shift()},this.getValues=function(e,t,r){var n;return null==this.values[e]?[]:null==this.values[e][t]?[]:(n=[],angular.forEach(this.values[e][t],function(e){return function(e,t){if(null!=e.values[r])return n.push({x:e.timestamp,y:e.values[r]})}}(this)),n)},this.setupLSFor=function(e,t){if(null==this.metrics[e]&&(this.metrics[e]={}),null==this.metrics[e][t])return this.metrics[e][t]=[]},this.addMetric=function(e,t,r){return this.setupLSFor(e,t),this.metrics[e][t].push({id:r,size:"small",view:"chart"}),this.saveSetup()},this.removeMet
 ric=function(e){return function(t,r,n){var i;if(null!=e.metrics[t][r])return i=e.metrics[t][r].indexOf(n),i===-1&&(i=_.findIndex(e.metrics[t][r],{id:n})),i!==-1&&e.metrics[t][r].splice(i,1),e.saveSetup()}}(this),this.setMetricSize=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:i,view:n.view}),e.saveSetup()}}(this),this.setMetricView=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:n.size,view:i}),e.saveSetup()}}(this),this.orderMetrics=function(e,t,r,n){return this.setupLSFor(e,t),angular.forEach(this.metrics[e][t],function(i){return function(o,a){if(o.id===r.id&&(i.metrics[e][t].splice(a,1),a<n))return n-=1}}(this)),this.metrics[e][t].splice(n,0,r),this.saveSetup()},this.getMetricsSe
 tup=function(e){return function(t,r){return{names:_.map(e.metrics[t][r],function(e){return _.isString(e)?{id:e,size:"small",view:"chart"}:e})}}}(this),this.getAvailableMetrics=function(n){return function(i,o){var a;return n.setupLSFor(i,o),a=t.defer(),e.get(r.jobServer+"jobs/"+i+"/vertices/"+o+"/metrics").success(function(e){var t;return t=[],angular.forEach(e,function(e,r){var a;if(a=n.metrics[i][o].indexOf(e.id),a===-1&&(a=_.findIndex(n.metrics[i][o],{id:e.id})),a===-1)return t.push(e)}),a.resolve(t)}),a.promise}}(this),this.getAllAvailableMetrics=function(n){return function(n,i){var o;return o=t.defer(),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics").success(function(e){return o.resolve(e)}),o.promise}}(this),this.getMetrics=function(n,i,o){var a,s;return a=t.defer(),s=o.join(","),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics?get="+s).success(function(e){return function(t){var r,o;return o={},angular.forEach(t,function(e,t){return o[e.id]=parseInt(e.value)}),r={time
 stamp:Date.now(),values:o},e.saveValue(n,i,r),a.resolve(r)}}(this)),a.promise},this.setupLS(),this}]),angular.module("flinkApp").controller("OverviewController",["$scope","OverviewService","JobsService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.jobObserver=function(){return e.runningJobs=r.getJobs("running"),e.finishedJobs=r.getJobs("finished")},r.registerObserver(e.jobObserver),e.$on("$destroy",function(){return r.unRegisterObserver(e.jobObserver)}),e.jobObserver(),t.loadOverview().then(function(t){return e.overview=t}),o=n(function(){return t.loadOverview().then(function(t){return e.overview=t})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]),angular.module("flinkApp").service("OverviewService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadOverview=function(){var i;return i=r.defer(),e.get(t.jobServer+"overview").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").
 controller("JobSubmitController",["$scope","JobSubmitService","$interval","flinkConfig","$state","$location",function(e,t,r,n,i,o){var a;return e.yarn=o.absUrl().indexOf("/proxy/application_")!==-1,e.loadList=function(){return t.loadJarList().then(function(t){return e.address=t.address,e.noaccess=t.error,e.jars=t.files})},e.defaultState=function(){return e.plan=null,e.error=null,e.state={selected:null,parallelism:"",savepointPath:"",allowNonRestoredState:!1,"entry-class":"","program-args":"","plan-button":"Show Plan","submit-button":"Submit","action-time":0}},e.defaultState(),e.uploader={},e.loadList(),a=r(function(){return e.loadList()},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(a)}),e.selectJar=function(t){return e.state.selected===t?e.defaultState():(e.defaultState(),e.state.selected=t)},e.deleteJar=function(r,n){return e.state.selected===n&&e.defaultState(),angular.element(r.currentTarget).removeClass("fa-remove").addClass("fa-spin fa-spinner"),t.deleteJa
 r(n).then(function(e){if(angular.element(r.currentTarget).removeClass("fa-spin fa-spinner").addClass("fa-remove"),null!=e.error)return alert(e.error)})},e.loadEntryClass=function(t){return e.state["entry-class"]=t},e.getPlan=function(){var r;if("Show Plan"===e.state["plan-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submit",e.state["plan-button"]="Getting Plan",e.error=null,e.plan=null,t.getPlan(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"]}).then(function(t){if(r===e.state["action-time"])return e.state["plan-button"]="Show Plan",e.error=t.error,e.plan=t.plan})},e.runJob=function(){var r;if("Submit"===e.state["submit-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submitting",e.state["plan-button"]="Show Plan",e.error=null,t.runJob(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"
 program-args":e.state["program-args"],savepointPath:e.state.savepointPath,allowNonRestoredState:e.state.allowNonRestoredState}).then(function(t){if(r===e.state["action-time"]&&(e.state["submit-button"]="Submit",e.error=t.error,null!=t.jobid))return i.go

<TRUNCATED>

[12/22] flink git commit: [FLINK-8613] [flip6] [yarn] Return excess containers

Posted by tr...@apache.org.
[FLINK-8613] [flip6] [yarn] Return excess containers

Upon notification of newly allocated containers, the YarnResourceManager
will only accept as many containers as there are pending container requests.
All excess containers will be returned.

This closes #5436.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/879185c8
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/879185c8
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/879185c8

Branch: refs/heads/master
Commit: 879185c8385047b6557db07bc752b95fa7ba9fa0
Parents: c9bc974
Author: Till Rohrmann <tr...@apache.org>
Authored: Wed Feb 7 17:00:40 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Thu Feb 15 08:44:20 2018 +0100

----------------------------------------------------------------------
 .../apache/flink/yarn/YarnResourceManager.java  | 58 +++++++++++++-------
 1 file changed, 37 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/879185c8/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java
index 8104d49..305a2f5 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java
@@ -325,26 +325,43 @@ public class YarnResourceManager extends ResourceManager<YarnWorkerNode> impleme
 	@Override
 	public void onContainersAllocated(List<Container> containers) {
 		for (Container container : containers) {
-			numPendingContainerRequests = Math.max(0, numPendingContainerRequests - 1);
-			log.info("Received new container: {} - Remaining pending container requests: {}",
-					container.getId(), numPendingContainerRequests);
-			final String containerIdStr = container.getId().toString();
-			workerNodeMap.put(new ResourceID(containerIdStr),
-					new YarnWorkerNode(container));
-			try {
-				/** Context information used to start a TaskExecutor Java process */
-				ContainerLaunchContext taskExecutorLaunchContext =
-						createTaskExecutorLaunchContext(
-								container.getResource(), containerIdStr, container.getNodeId().getHost());
-				nodeManagerClient.startContainer(container, taskExecutorLaunchContext);
-			}
-			catch (Throwable t) {
-				// failed to launch the container, will release the failed one and ask for a new one
-				log.error("Could not start TaskManager in container {},", container, t);
+			log.info(
+				"Received new container: {} - Remaining pending container requests: {}",
+				container.getId(),
+				numPendingContainerRequests);
+
+			if (numPendingContainerRequests > 0) {
+				numPendingContainerRequests--;
+
+				final String containerIdStr = container.getId().toString();
+
+				workerNodeMap.put(new ResourceID(containerIdStr), new YarnWorkerNode(container));
+
+				try {
+					// Context information used to start a TaskExecutor Java process
+					ContainerLaunchContext taskExecutorLaunchContext = createTaskExecutorLaunchContext(
+						container.getResource(),
+						containerIdStr,
+						container.getNodeId().getHost());
+
+					nodeManagerClient.startContainer(container, taskExecutorLaunchContext);
+				} catch (Throwable t) {
+					log.error("Could not start TaskManager in container {}.", container.getId(), t);
+
+					// release the failed container
+					resourceManagerClient.releaseAssignedContainer(container.getId());
+					// and ask for a new one
+					requestYarnContainer(container.getResource(), container.getPriority());
+				}
+			} else {
+				// return the excessive containers
+				log.info("Returning excess container {}.", container.getId());
 				resourceManagerClient.releaseAssignedContainer(container.getId());
-				requestYarnContainer(container.getResource(), container.getPriority());
 			}
 		}
+
+		// if we are waiting for no further containers, we can go to the
+		// regular heartbeat interval
 		if (numPendingContainerRequests <= 0) {
 			resourceManagerClient.setHeartbeatInterval(yarnHeartbeatIntervalMillis);
 		}
@@ -403,14 +420,13 @@ public class YarnResourceManager extends ResourceManager<YarnWorkerNode> impleme
 	}
 
 	private void requestYarnContainer(Resource resource, Priority priority) {
-		resourceManagerClient.addContainerRequest(
-				new AMRMClient.ContainerRequest(resource, null, null, priority));
+		resourceManagerClient.addContainerRequest(new AMRMClient.ContainerRequest(resource, null, null, priority));
+
 		// make sure we transmit the request fast and receive fast news of granted allocations
 		resourceManagerClient.setHeartbeatInterval(FAST_YARN_HEARTBEAT_INTERVAL_MS);
 
 		numPendingContainerRequests++;
-		log.info("Requesting new TaskManager container pending requests: {}",
-				numPendingContainerRequests);
+		log.info("Requesting new TaskManager container pending requests: {}", numPendingContainerRequests);
 	}
 
 	private ContainerLaunchContext createTaskExecutorLaunchContext(Resource resource, String containerId, String host)


[02/22] flink git commit: [FLINK-8529] [flip6] Let Yarn entry points use APPLICATION_MASTER_PORT

Posted by tr...@apache.org.
[FLINK-8529] [flip6] Let Yarn entry points use APPLICATION_MASTER_PORT

Let all Yarn entry points use the YarnConfigOptions.APPLICATION_MASTER_PORT option
to specify the valid port range for the common RpcService.

This closes #5388.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/42f49ac9
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/42f49ac9
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/42f49ac9

Branch: refs/heads/master
Commit: 42f49ac9172fe5c3ff4cc45ccf5d82142351e4e2
Parents: c7d2a0d
Author: Till Rohrmann <tr...@apache.org>
Authored: Tue Jan 30 15:15:49 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Thu Feb 15 08:44:17 2018 +0100

----------------------------------------------------------------------
 .../runtime/entrypoint/ClusterEntrypoint.java     | 18 ++++++++++++++++--
 .../yarn/entrypoint/YarnJobClusterEntrypoint.java |  6 ++++++
 .../entrypoint/YarnSessionClusterEntrypoint.java  |  6 ++++++
 3 files changed, 28 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/42f49ac9/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java
index 156efdc..dfb1b4c 100755
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java
@@ -172,10 +172,14 @@ public abstract class ClusterEntrypoint implements FatalErrorHandler {
 		LOG.info("Initializing cluster services.");
 
 		final String bindAddress = configuration.getString(JobManagerOptions.ADDRESS);
-		// TODO: Add support for port ranges
-		final String portRange = String.valueOf(configuration.getInteger(JobManagerOptions.PORT));
+		final String portRange = getRPCPortRange(configuration);
 
 		commonRpcService = createRpcService(configuration, bindAddress, portRange);
+
+		// update the configuration used to create the high availability services
+		configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress());
+		configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort());
+
 		haServices = createHaServices(configuration, commonRpcService.getExecutor());
 		blobServer = new BlobServer(configuration, haServices.createBlobStore());
 		blobServer.start();
@@ -188,6 +192,16 @@ public abstract class ClusterEntrypoint implements FatalErrorHandler {
 		metricRegistry.startQueryService(actorSystem, null);
 	}
 
+	/**
+	 * Returns the port range for the common {@link RpcService}.
+	 *
+	 * @param configuration to extract the port range from
+	 * @return Port range for the common {@link RpcService}
+	 */
+	protected String getRPCPortRange(Configuration configuration) {
+		return String.valueOf(configuration.getInteger(JobManagerOptions.PORT));
+	}
+
 	protected RpcService createRpcService(
 			Configuration configuration,
 			String bindAddress,

http://git-wip-us.apache.org/repos/asf/flink/blob/42f49ac9/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnJobClusterEntrypoint.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnJobClusterEntrypoint.java b/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnJobClusterEntrypoint.java
index 4a9442a..ca4cb9c 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnJobClusterEntrypoint.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnJobClusterEntrypoint.java
@@ -39,6 +39,7 @@ import org.apache.flink.runtime.util.SignalHandler;
 import org.apache.flink.util.FlinkException;
 import org.apache.flink.util.Preconditions;
 import org.apache.flink.yarn.YarnResourceManager;
+import org.apache.flink.yarn.configuration.YarnConfigOptions;
 
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 
@@ -75,6 +76,11 @@ public class YarnJobClusterEntrypoint extends JobClusterEntrypoint {
 	}
 
 	@Override
+	protected String getRPCPortRange(Configuration configuration) {
+		return configuration.getString(YarnConfigOptions.APPLICATION_MASTER_PORT);
+	}
+
+	@Override
 	protected ResourceManager<?> createResourceManager(
 			Configuration configuration,
 			ResourceID resourceId,

http://git-wip-us.apache.org/repos/asf/flink/blob/42f49ac9/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnSessionClusterEntrypoint.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnSessionClusterEntrypoint.java b/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnSessionClusterEntrypoint.java
index 64b4aca..232467a 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnSessionClusterEntrypoint.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnSessionClusterEntrypoint.java
@@ -37,6 +37,7 @@ import org.apache.flink.runtime.util.JvmShutdownSafeguard;
 import org.apache.flink.runtime.util.SignalHandler;
 import org.apache.flink.util.Preconditions;
 import org.apache.flink.yarn.YarnResourceManager;
+import org.apache.flink.yarn.configuration.YarnConfigOptions;
 
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 
@@ -65,6 +66,11 @@ public class YarnSessionClusterEntrypoint extends SessionClusterEntrypoint {
 	}
 
 	@Override
+	protected String getRPCPortRange(Configuration configuration) {
+		return configuration.getString(YarnConfigOptions.APPLICATION_MASTER_PORT);
+	}
+
+	@Override
 	protected ResourceManager<?> createResourceManager(
 			Configuration configuration,
 			ResourceID resourceId,


[21/22] flink git commit: [FLINK-8643] [flip6] Use JobManagerOptions#SLOT_REQUEST_TIMEOUT in ExecutionGraph

Posted by tr...@apache.org.
[FLINK-8643] [flip6] Use JobManagerOptions#SLOT_REQUEST_TIMEOUT in ExecutionGraph

This commit changes the initialization of the ExecutionGraph to use the
JobManagerOptions#SLOT_REQUEST_TIMEOUT for the slot allocation. Furthermore,
it changes the behaviour of the SlotPool#ProviderAndOwner implementation such
that the timeout is given to it via the SlotProvider#allocateSlot call.

This closes #5475.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1321c170
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1321c170
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1321c170

Branch: refs/heads/master
Commit: 1321c17047daed756ce4216814b8708162a57e83
Parents: 5ed180c
Author: Till Rohrmann <tr...@apache.org>
Authored: Tue Feb 13 12:41:44 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Thu Feb 15 08:46:14 2018 +0100

----------------------------------------------------------------------
 .../flink/runtime/executiongraph/Execution.java |  59 ++---
 .../runtime/executiongraph/ExecutionGraph.java  |  97 ++++----
 .../executiongraph/ExecutionGraphBuilder.java   |   8 +-
 .../executiongraph/ExecutionJobVertex.java      |   7 +-
 .../runtime/jobmanager/scheduler/Scheduler.java |   7 +-
 .../flink/runtime/jobmaster/JobMaster.java      |   6 +-
 .../runtime/jobmaster/slotpool/SlotPool.java    |  39 ++--
 .../jobmaster/slotpool/SlotProvider.java        |   9 +-
 .../flink/runtime/jobmanager/JobManager.scala   |   4 +
 .../CheckpointSettingsSerializableTest.java     |   7 +-
 ...ExecutionGraphCheckpointCoordinatorTest.java |   6 +-
 .../ExecutionGraphDeploymentTest.java           |  13 +-
 .../ExecutionGraphMetricsTest.java              |   2 +-
 .../ExecutionGraphRescalingTest.java            |   4 +
 .../ExecutionGraphSchedulingTest.java           |  37 ++--
 .../executiongraph/ExecutionGraphTestUtils.java |   1 +
 .../runtime/executiongraph/ExecutionTest.java   |   9 +-
 .../ExecutionVertexLocalityTest.java            |  13 +-
 .../ExecutionVertexSchedulingTest.java          |  14 +-
 .../PipelinedRegionFailoverConcurrencyTest.java |   6 +-
 .../executiongraph/ProgrammedSlotProvider.java  |   8 +-
 .../RestartPipelinedRegionStrategyTest.java     |  12 +-
 .../PipelinedFailoverRegionBuildingTest.java    |   4 +-
 .../utils/SimpleSlotProvider.java               |   4 +-
 .../ScheduleWithCoLocationHintTest.java         | 121 +++++-----
 .../scheduler/SchedulerIsolatedTasksTest.java   |  44 ++--
 .../scheduler/SchedulerSlotSharingTest.java     | 222 +++++++++----------
 .../jobmanager/scheduler/SchedulerTest.java     |  44 ++++
 .../jobmanager/scheduler/SchedulerTestBase.java |   9 +-
 .../slotpool/SlotPoolCoLocationTest.java        |  13 +-
 .../jobmaster/slotpool/SlotPoolRpcTest.java     |  18 +-
 .../slotpool/SlotPoolSlotSharingTest.java       |  31 ++-
 .../jobmaster/slotpool/SlotPoolTest.java        |  33 ++-
 .../partitioner/RescalePartitionerTest.java     |   3 +-
 34 files changed, 521 insertions(+), 393 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index a151074..0275004 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -133,7 +133,7 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 
 	private final int attemptNumber;
 
-	private final Time timeout;
+	private final Time rpcTimeout;
 
 	private final ConcurrentLinkedQueue<PartialInputChannelDeploymentDescriptor> partialInputChannelDeploymentDescriptors;
 
@@ -180,8 +180,8 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 	 *             The global modification version of the execution graph when this execution was created
 	 * @param startTimestamp
 	 *             The timestamp that marks the creation of this Execution
-	 * @param timeout
-	 *             The timeout for RPC calls like deploy/cancel/stop.
+	 * @param rpcTimeout
+	 *             The rpcTimeout for RPC calls like deploy/cancel/stop.
 	 */
 	public Execution(
 			Executor executor,
@@ -189,12 +189,12 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 			int attemptNumber,
 			long globalModVersion,
 			long startTimestamp,
-			Time timeout) {
+			Time rpcTimeout) {
 
 		this.executor = checkNotNull(executor);
 		this.vertex = checkNotNull(vertex);
 		this.attemptId = new ExecutionAttemptID();
-		this.timeout = checkNotNull(timeout);
+		this.rpcTimeout = checkNotNull(rpcTimeout);
 
 		this.globalModVersion = globalModVersion;
 		this.attemptNumber = attemptNumber;
@@ -359,9 +359,13 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 	// --------------------------------------------------------------------------------------------
 
 	public boolean scheduleForExecution() {
-		SlotProvider resourceProvider = getVertex().getExecutionGraph().getSlotProvider();
-		boolean allowQueued = getVertex().getExecutionGraph().isQueuedSchedulingAllowed();
-		return scheduleForExecution(resourceProvider, allowQueued, LocationPreferenceConstraint.ANY);
+		final ExecutionGraph executionGraph = getVertex().getExecutionGraph();
+		final SlotProvider resourceProvider = executionGraph.getSlotProvider();
+		final boolean allowQueued = executionGraph.isQueuedSchedulingAllowed();
+		return scheduleForExecution(
+			resourceProvider,
+			allowQueued,
+			LocationPreferenceConstraint.ANY);
 	}
 
 	/**
@@ -373,18 +377,19 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 	 * @param queued Flag to indicate whether the scheduler may queue this task if it cannot
 	 *               immediately deploy it.
 	 * @param locationPreferenceConstraint constraint for the location preferences
-	 * 
 	 * @throws IllegalStateException Thrown, if the vertex is not in CREATED state, which is the only state that permits scheduling.
 	 */
 	public boolean scheduleForExecution(
-		SlotProvider slotProvider,
-		boolean queued,
-		LocationPreferenceConstraint locationPreferenceConstraint) {
+			SlotProvider slotProvider,
+			boolean queued,
+			LocationPreferenceConstraint locationPreferenceConstraint) {
 		try {
+			final Time allocationTimeout = vertex.getExecutionGraph().getAllocationTimeout();
 			final CompletableFuture<Execution> allocationFuture = allocateAndAssignSlotForExecution(
 				slotProvider,
 				queued,
-				locationPreferenceConstraint);
+				locationPreferenceConstraint,
+				allocationTimeout);
 
 			// IMPORTANT: We have to use the synchronous handle operation (direct executor) here so
 			// that we directly deploy the tasks if the slot allocation future is completed. This is
@@ -407,7 +412,7 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 
 			// if tasks have to scheduled immediately check that the task has been deployed
 			if (!queued && !deploymentFuture.isDone()) {
-				markFailed(new IllegalArgumentException("The slot allocation future has not been completed yet."));
+				allocationFuture.completeExceptionally(new IllegalArgumentException("The slot allocation future has not been completed yet."));
 			}
 
 			return true;
@@ -423,6 +428,7 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 	 * @param slotProvider to obtain a new slot from
 	 * @param queued if the allocation can be queued
 	 * @param locationPreferenceConstraint constraint for the location preferences
+	 * @param allocationTimeout rpcTimeout for allocating a new slot
 	 * @return Future which is completed with this execution once the slot has been assigned
 	 * 			or with an exception if an error occurred.
 	 * @throws IllegalExecutionStateException if this method has been called while not being in the CREATED state
@@ -430,7 +436,8 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 	public CompletableFuture<Execution> allocateAndAssignSlotForExecution(
 			SlotProvider slotProvider,
 			boolean queued,
-			LocationPreferenceConstraint locationPreferenceConstraint) throws IllegalExecutionStateException {
+			LocationPreferenceConstraint locationPreferenceConstraint,
+			Time allocationTimeout) throws IllegalExecutionStateException {
 
 		checkNotNull(slotProvider);
 
@@ -461,7 +468,8 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 						slotProvider.allocateSlot(
 							toSchedule,
 							queued,
-							preferredLocations))
+							preferredLocations,
+							allocationTimeout))
 				.thenApply(
 					(LogicalSlot logicalSlot) -> {
 						if (tryAssignResource(logicalSlot)) {
@@ -492,7 +500,7 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 
 		// Check if the TaskManager died in the meantime
 		// This only speeds up the response to TaskManagers failing concurrently to deployments.
-		// The more general check is the timeout of the deployment call
+		// The more general check is the rpcTimeout of the deployment call
 		if (!slot.isAlive()) {
 			throw new JobException("Target slot (TaskManager) for deployment is no longer alive.");
 		}
@@ -537,7 +545,7 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 
 			final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway();
 
-			final CompletableFuture<Acknowledge> submitResultFuture = taskManagerGateway.submitTask(deployment, timeout);
+			final CompletableFuture<Acknowledge> submitResultFuture = taskManagerGateway.submitTask(deployment, rpcTimeout);
 
 			submitResultFuture.whenCompleteAsync(
 				(ack, failure) -> {
@@ -548,7 +556,7 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 
 							markFailed(new Exception(
 								"Cannot deploy task " + taskname + " - TaskManager (" + getAssignedResourceLocation()
-									+ ") not responding after a timeout of " + timeout, failure));
+									+ ") not responding after a rpcTimeout of " + rpcTimeout, failure));
 						} else {
 							markFailed(failure);
 						}
@@ -572,7 +580,7 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 			final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway();
 
 			CompletableFuture<Acknowledge> stopResultFuture = FutureUtils.retry(
-				() -> taskManagerGateway.stopTask(attemptId, timeout),
+				() -> taskManagerGateway.stopTask(attemptId, rpcTimeout),
 				NUM_STOP_CALL_TRIES,
 				executor);
 
@@ -684,9 +692,10 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 				CompletableFuture.supplyAsync(
 					() -> {
 						try {
+							final ExecutionGraph executionGraph = consumerVertex.getExecutionGraph();
 							consumerVertex.scheduleForExecution(
-								consumerVertex.getExecutionGraph().getSlotProvider(),
-								consumerVertex.getExecutionGraph().isQueuedSchedulingAllowed(),
+								executionGraph.getSlotProvider(),
+								executionGraph.isQueuedSchedulingAllowed(),
 								LocationPreferenceConstraint.ANY); // there must be at least one known location
 						} catch (Throwable t) {
 							consumerVertex.fail(new IllegalStateException("Could not schedule consumer " +
@@ -698,7 +707,7 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 					executor);
 
 				// double check to resolve race conditions
-				if(consumerVertex.getExecutionState() == RUNNING){
+				if (consumerVertex.getExecutionState() == RUNNING){
 					consumerVertex.sendPartitionInfos();
 				}
 			}
@@ -1128,7 +1137,7 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 			final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway();
 
 			CompletableFuture<Acknowledge> cancelResultFuture = FutureUtils.retry(
-				() -> taskManagerGateway.cancelTask(attemptId, timeout),
+				() -> taskManagerGateway.cancelTask(attemptId, rpcTimeout),
 				NUM_CANCEL_CALL_TRIES,
 				executor);
 
@@ -1167,7 +1176,7 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 			final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway();
 			final TaskManagerLocation taskManagerLocation = slot.getTaskManagerLocation();
 
-			CompletableFuture<Acknowledge> updatePartitionsResultFuture = taskManagerGateway.updatePartitions(attemptId, partitionInfos, timeout);
+			CompletableFuture<Acknowledge> updatePartitionsResultFuture = taskManagerGateway.updatePartitions(attemptId, partitionInfos, rpcTimeout);
 
 			updatePartitionsResultFuture.whenCompleteAsync(
 				(ack, failure) -> {

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
index 188f6da..9331b2e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
@@ -95,7 +95,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLongFieldUpdater;
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
@@ -213,11 +213,10 @@ public class ExecutionGraph implements AccessExecutionGraph {
 	private final long[] stateTimestamps;
 
 	/** The timeout for all messages that require a response/acknowledgement */
-	private final Time rpcCallTimeout;
+	private final Time rpcTimeout;
 
-	/** The timeout for bulk slot allocation (eager scheduling mode). After this timeout,
-	 * slots are released and a recovery is triggered */
-	private final Time scheduleAllocationTimeout;
+	/** The timeout for slot allocations. */
+	private final Time allocationTimeout;
 
 	/** Strategy to use for restarts */
 	private final RestartStrategy restartStrategy;
@@ -356,19 +355,21 @@ public class ExecutionGraph implements AccessExecutionGraph {
 			failoverStrategy,
 			slotProvider,
 			ExecutionGraph.class.getClassLoader(),
-			VoidBlobWriter.getInstance());
+			VoidBlobWriter.getInstance(),
+			timeout);
 	}
 
 	public ExecutionGraph(
 			JobInformation jobInformation,
 			ScheduledExecutorService futureExecutor,
 			Executor ioExecutor,
-			Time timeout,
+			Time rpcTimeout,
 			RestartStrategy restartStrategy,
 			FailoverStrategy.Factory failoverStrategyFactory,
 			SlotProvider slotProvider,
 			ClassLoader userClassLoader,
-			BlobWriter blobWriter) throws IOException {
+			BlobWriter blobWriter,
+			Time allocationTimeout) throws IOException {
 
 		checkNotNull(futureExecutor);
 
@@ -395,8 +396,8 @@ public class ExecutionGraph implements AccessExecutionGraph {
 		this.stateTimestamps = new long[JobStatus.values().length];
 		this.stateTimestamps[JobStatus.CREATED.ordinal()] = System.currentTimeMillis();
 
-		this.rpcCallTimeout = checkNotNull(timeout);
-		this.scheduleAllocationTimeout = checkNotNull(timeout);
+		this.rpcTimeout = checkNotNull(rpcTimeout);
+		this.allocationTimeout = checkNotNull(allocationTimeout);
 
 		this.restartStrategy = restartStrategy;
 		this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());
@@ -439,6 +440,10 @@ public class ExecutionGraph implements AccessExecutionGraph {
 		return scheduleMode;
 	}
 
+	public Time getAllocationTimeout() {
+		return allocationTimeout;
+	}
+
 	@Override
 	public boolean isArchived() {
 		return false;
@@ -821,7 +826,7 @@ public class ExecutionGraph implements AccessExecutionGraph {
 				this,
 				jobVertex,
 				1,
-				rpcCallTimeout,
+				rpcTimeout,
 				globalModVersion,
 				createTimestamp);
 
@@ -861,7 +866,7 @@ public class ExecutionGraph implements AccessExecutionGraph {
 					break;
 
 				case EAGER:
-					scheduleEager(slotProvider, scheduleAllocationTimeout);
+					scheduleEager(slotProvider, allocationTimeout);
 					break;
 
 				default:
@@ -910,59 +915,45 @@ public class ExecutionGraph implements AccessExecutionGraph {
 			Collection<CompletableFuture<Execution>> allocationFutures = ejv.allocateResourcesForAll(
 				slotProvider,
 				queued,
-				LocationPreferenceConstraint.ALL);
+				LocationPreferenceConstraint.ALL,
+				allocationTimeout);
 
 			allAllocationFutures.addAll(allocationFutures);
 		}
 
 		// this future is complete once all slot futures are complete.
 		// the future fails once one slot future fails.
-		final ConjunctFuture<Collection<Execution>> allAllocationsComplete = FutureUtils.combineAll(allAllocationFutures);
-
-		// make sure that we fail if the allocation timeout was exceeded
-		final ScheduledFuture<?> timeoutCancelHandle = futureExecutor.schedule(new Runnable() {
-			@Override
-			public void run() {
-				// When the timeout triggers, we try to complete the conjunct future with an exception.
-				// Note that this is a no-op if the future is already completed
-				int numTotal = allAllocationsComplete.getNumFuturesTotal();
-				int numComplete = allAllocationsComplete.getNumFuturesCompleted();
-				String message = "Could not allocate all requires slots within timeout of " +
-						timeout + ". Slots required: " + numTotal + ", slots allocated: " + numComplete;
-
-				allAllocationsComplete.completeExceptionally(new NoResourceAvailableException(message));
-			}
-		}, timeout.getSize(), timeout.getUnit());
-
-
-		allAllocationsComplete.handleAsync(
-			(Collection<Execution> executions, Throwable throwable) -> {
-				try {
-					// we do not need the cancellation timeout any more
-					timeoutCancelHandle.cancel(false);
+		final ConjunctFuture<Collection<Execution>> allAllocationsFuture = FutureUtils.combineAll(allAllocationFutures);
+
+		allAllocationsFuture.whenCompleteAsync(
+			(Collection<Execution> allAllocations, Throwable throwable) -> {
+				if (throwable != null) {
+					final Throwable strippedThrowable = ExceptionUtils.stripCompletionException(throwable);
+					final Throwable resultThrowable;
+
+					if (strippedThrowable instanceof TimeoutException) {
+						int numTotal = allAllocationsFuture.getNumFuturesTotal();
+						int numComplete = allAllocationsFuture.getNumFuturesCompleted();
+						String message = "Could not allocate all requires slots within timeout of " +
+							timeout + ". Slots required: " + numTotal + ", slots allocated: " + numComplete;
+
+						resultThrowable = new NoResourceAvailableException(message);
+					} else {
+						resultThrowable = strippedThrowable;
+					}
 
-					if (throwable == null) {
+					failGlobal(resultThrowable);
+				} else {
+					try {
 						// successfully obtained all slots, now deploy
-						for (Execution execution : executions) {
+						for (Execution execution : allAllocations) {
 							execution.deploy();
 						}
-					}
-					else {
-						// let the exception handler deal with this
-						throw throwable;
+					} catch (Throwable t) {
+						failGlobal(new FlinkException("Could not deploy executions.", t));
 					}
 				}
-				catch (Throwable t) {
-					// we catch everything here to make sure cleanup happens and the
-					// ExecutionGraph notices the error
-					failGlobal(ExceptionUtils.stripCompletionException(t));
-				}
-
-				// Wouldn't it be nice if we could return an actual Void object?
-				// return (Void) Unsafe.getUnsafe().allocateInstance(Void.class);
-				return null;
-			},
-			futureExecutor);
+			}, futureExecutor);
 	}
 
 	public void cancel() {

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
index cd719be..5d076e2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraphBuilder.java
@@ -87,11 +87,12 @@ public class ExecutionGraphBuilder {
 			SlotProvider slotProvider,
 			ClassLoader classLoader,
 			CheckpointRecoveryFactory recoveryFactory,
-			Time timeout,
+			Time rpcTimeout,
 			RestartStrategy restartStrategy,
 			MetricGroup metrics,
 			int parallelismForAutoMax,
 			BlobWriter blobWriter,
+			Time allocationTimeout,
 			Logger log)
 		throws JobExecutionException, JobException {
 
@@ -119,12 +120,13 @@ public class ExecutionGraphBuilder {
 					jobInformation,
 					futureExecutor,
 					ioExecutor,
-					timeout,
+					rpcTimeout,
 					restartStrategy,
 					failoverStrategy,
 					slotProvider,
 					classLoader,
-					blobWriter);
+					blobWriter,
+					allocationTimeout);
 		} catch (IOException e) {
 			throw new JobException("Could not create the ExecutionGraph.", e);
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
index e07c71e..db336f5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
@@ -496,11 +496,13 @@ public class ExecutionJobVertex implements AccessExecutionJobVertex, Archiveable
 	 * @param resourceProvider The resource provider from whom the slots are requested.
 	 * @param queued if the allocation can be queued
 	 * @param locationPreferenceConstraint constraint for the location preferences
+	 * @param allocationTimeout timeout for allocating the individual slots
 	 */
 	public Collection<CompletableFuture<Execution>> allocateResourcesForAll(
 			SlotProvider resourceProvider,
 			boolean queued,
-			LocationPreferenceConstraint locationPreferenceConstraint) {
+			LocationPreferenceConstraint locationPreferenceConstraint,
+			Time allocationTimeout) {
 		final ExecutionVertex[] vertices = this.taskVertices;
 		final CompletableFuture<Execution>[] slots = new CompletableFuture[vertices.length];
 
@@ -512,7 +514,8 @@ public class ExecutionJobVertex implements AccessExecutionJobVertex, Archiveable
 			final CompletableFuture<Execution> allocationFuture = exec.allocateAndAssignSlotForExecution(
 				resourceProvider,
 				queued,
-				locationPreferenceConstraint);
+				locationPreferenceConstraint,
+				allocationTimeout);
 			slots[i] = allocationFuture;
 		}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/Scheduler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/Scheduler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/Scheduler.java
index 40fb760..77f89b7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/Scheduler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/Scheduler.java
@@ -19,6 +19,7 @@
 package org.apache.flink.runtime.jobmanager.scheduler;
 
 import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.time.Time;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.concurrent.FutureUtils;
 import org.apache.flink.runtime.executiongraph.ExecutionVertex;
@@ -60,6 +61,7 @@ import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executor;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
 
 /**
  * The scheduler is responsible for distributing the ready-to-run tasks among instances and slots.
@@ -142,7 +144,8 @@ public class Scheduler implements InstanceListener, SlotAvailabilityListener, Sl
 	public CompletableFuture<LogicalSlot> allocateSlot(
 			ScheduledUnit task,
 			boolean allowQueued,
-			Collection<TaskManagerLocation> preferredLocations) {
+			Collection<TaskManagerLocation> preferredLocations,
+			Time allocationTimeout) {
 
 		try {
 			final Object ret = scheduleTask(task, allowQueued, preferredLocations);
@@ -153,7 +156,7 @@ public class Scheduler implements InstanceListener, SlotAvailabilityListener, Sl
 			else if (ret instanceof CompletableFuture) {
 				@SuppressWarnings("unchecked")
 				CompletableFuture<LogicalSlot> typed = (CompletableFuture<LogicalSlot>) ret;
-				return typed;
+				return FutureUtils.orTimeout(typed, allocationTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);
 			}
 			else {
 				// this should never happen, simply guard this case with an exception

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index e7583d0..6c5e524 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -281,12 +281,13 @@ public class JobMaster extends FencedRpcEndpoint<JobMasterId> implements JobMast
 			jobGraph.getJobID(),
 			SystemClock.getInstance(),
 			Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT)),
-			Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_ALLOCATION_RM_TIMEOUT)),
-			Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_RM_TIMEOUT)),
+			rpcTimeout,
 			Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_IDLE_TIMEOUT)));
 
 		this.slotPoolGateway = slotPool.getSelfGateway(SlotPoolGateway.class);
 
+		final Time allocationTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));
+
 		this.executionGraph = ExecutionGraphBuilder.buildGraph(
 			null,
 			jobGraph,
@@ -301,6 +302,7 @@ public class JobMaster extends FencedRpcEndpoint<JobMasterId> implements JobMast
 			jobMetricGroup,
 			-1,
 			blobServer,
+			allocationTimeout,
 			log);
 
 		// register self as job status change listener

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java
index 12a6012..59f382f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java
@@ -108,10 +108,10 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 	private final HashMap<SlotRequestId, PendingRequest> waitingForResourceManager;
 
 	/** Timeout for external request calls (e.g. to the ResourceManager or the TaskExecutor). */
-	private final Time timeout;
+	private final Time rpcTimeout;
 
 	/** Timeout for allocation round trips (RM -> launch TM -> offer slot). */
-	private final Time resourceManagerAllocationTimeout;
+	private final Time slotRequestTimeout;
 
 	/** Timeout for releasing idle slots. */
 	private final Time idleSlotTimeout;
@@ -137,7 +137,6 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 			rpcService,
 			jobId,
 			SystemClock.getInstance(),
-			Time.milliseconds(JobManagerOptions.SLOT_REQUEST_TIMEOUT.defaultValue()),
 			Time.milliseconds(JobManagerOptions.SLOT_ALLOCATION_RM_TIMEOUT.defaultValue()),
 			Time.milliseconds(JobManagerOptions.SLOT_REQUEST_RM_TIMEOUT.defaultValue()),
 			Time.milliseconds(JobManagerOptions.SLOT_IDLE_TIMEOUT.defaultValue()));
@@ -148,17 +147,16 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 			JobID jobId,
 			Clock clock,
 			Time slotRequestTimeout,
-			Time resourceManagerAllocationTimeout,
-			Time resourceManagerRequestTimeout,
+			Time rpcTimeout,
 			Time idleSlotTimeout) {
 
 		super(rpcService);
 
 		this.jobId = checkNotNull(jobId);
 		this.clock = checkNotNull(clock);
-		this.timeout = checkNotNull(resourceManagerRequestTimeout);
+		this.rpcTimeout = checkNotNull(rpcTimeout);
 		this.idleSlotTimeout = checkNotNull(idleSlotTimeout);
-		this.resourceManagerAllocationTimeout = checkNotNull(resourceManagerAllocationTimeout);
+		this.slotRequestTimeout = checkNotNull(slotRequestTimeout);
 
 		this.registeredTaskManagers = new HashSet<>(16);
 		this.allocatedSlots = new AllocatedSlots();
@@ -166,7 +164,7 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 		this.pendingRequests = new DualKeyMap<>(16);
 		this.waitingForResourceManager = new HashMap<>(16);
 
-		this.providerAndOwner = new ProviderAndOwner(getSelfGateway(SlotPoolGateway.class), slotRequestTimeout);
+		this.providerAndOwner = new ProviderAndOwner(getSelfGateway(SlotPoolGateway.class));
 
 		this.slotSharingManagers = new HashMap<>(4);
 
@@ -677,7 +675,7 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 		CompletableFuture<Acknowledge> rmResponse = resourceManagerGateway.requestSlot(
 			jobMasterId,
 			new SlotRequest(jobId, allocationId, pendingRequest.getResourceProfile(), jobManagerAddress),
-			timeout);
+			rpcTimeout);
 
 		CompletableFuture<Void> slotRequestProcessingFuture = rmResponse.thenAcceptAsync(
 			(Acknowledge value) -> {
@@ -698,7 +696,7 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 	private void slotRequestToResourceManagerSuccess(final SlotRequestId requestId) {
 		// a request is pending from the ResourceManager to a (future) TaskManager
 		// we only add the watcher here in case that request times out
-		scheduleRunAsync(() -> checkTimeoutSlotAllocation(requestId), resourceManagerAllocationTimeout);
+		scheduleRunAsync(() -> checkTimeoutSlotAllocation(requestId), slotRequestTimeout);
 	}
 
 	private void slotRequestToResourceManagerFailed(SlotRequestId slotRequestID, Throwable failure) {
@@ -727,12 +725,7 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 
 		waitingForResourceManager.put(pendingRequest.getSlotRequestId(), pendingRequest);
 
-		scheduleRunAsync(new Runnable() {
-			@Override
-			public void run() {
-				checkTimeoutRequestWaitingForResourceManager(pendingRequest.getSlotRequestId());
-			}
-		}, timeout);
+		scheduleRunAsync(() -> checkTimeoutRequestWaitingForResourceManager(pendingRequest.getSlotRequestId()), slotRequestTimeout);
 	}
 
 	private void checkTimeoutRequestWaitingForResourceManager(SlotRequestId slotRequestId) {
@@ -1000,7 +993,7 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 
 	/**
 	 * Fail the specified allocation and release the corresponding slot if we have one.
-	 * This may triggered by JobManager when some slot allocation failed with timeout.
+	 * This may triggered by JobManager when some slot allocation failed with rpcTimeout.
 	 * Or this could be triggered by TaskManager, when it finds out something went wrong with the slot,
 	 * and decided to take it back.
 	 *
@@ -1081,7 +1074,7 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 
 		for (AllocatedSlot removedSlot : removedSlots) {
 			TaskManagerGateway taskManagerGateway = removedSlot.getTaskManagerGateway();
-			taskManagerGateway.freeSlot(removedSlot.getAllocationId(), cause, timeout);
+			taskManagerGateway.freeSlot(removedSlot.getAllocationId(), cause, rpcTimeout);
 		}
 	}
 
@@ -1111,7 +1104,7 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 				final CompletableFuture<Acknowledge> freeSlotFuture = expiredSlot.getTaskManagerGateway().freeSlot(
 					allocationID,
 					cause,
-					timeout);
+					rpcTimeout);
 
 				freeSlotFuture.whenCompleteAsync(
 					(Acknowledge ignored, Throwable throwable) -> {
@@ -1536,11 +1529,8 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 
 		private final SlotPoolGateway gateway;
 
-		private final Time timeout;
-
-		ProviderAndOwner(SlotPoolGateway gateway, Time timeout) {
+		ProviderAndOwner(SlotPoolGateway gateway) {
 			this.gateway = gateway;
-			this.timeout = timeout;
 		}
 
 		@Override
@@ -1558,7 +1548,8 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 		public CompletableFuture<LogicalSlot> allocateSlot(
 				ScheduledUnit task,
 				boolean allowQueued,
-				Collection<TaskManagerLocation> preferredLocations) {
+				Collection<TaskManagerLocation> preferredLocations,
+				Time timeout) {
 
 			final SlotRequestId requestId = new SlotRequestId();
 			CompletableFuture<LogicalSlot> slotFuture = gateway.allocateSlot(

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotProvider.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotProvider.java
index 8e8d019..4f82688 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotProvider.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotProvider.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.runtime.jobmaster.slotpool;
 
+import org.apache.flink.api.common.time.Time;
 import org.apache.flink.runtime.jobmaster.LogicalSlot;
 import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
@@ -41,13 +42,15 @@ public interface SlotProvider {
 	/**
 	 * Allocating slot with specific requirement.
 	 *
-	 * @param task         The task to allocate the slot for
-	 * @param allowQueued  Whether allow the task be queued if we do not have enough resource
+	 * @param task The task to allocate the slot for
+	 * @param allowQueued Whether allow the task be queued if we do not have enough resource
 	 * @param preferredLocations preferred locations for the slot allocation
+	 * @param timeout after which the allocation fails with a timeout exception
 	 * @return The future of the allocation
 	 */
 	CompletableFuture<LogicalSlot> allocateSlot(
 		ScheduledUnit task,
 		boolean allowQueued,
-		Collection<TaskManagerLocation> preferredLocations);
+		Collection<TaskManagerLocation> preferredLocations,
+		Time timeout);
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index b795609..d0e401c 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -1268,6 +1268,9 @@ class JobManager(
             true
         }
 
+        val allocationTimeout: Long = flinkConfiguration.getLong(
+          JobManagerOptions.SLOT_REQUEST_TIMEOUT)
+
         executionGraph = ExecutionGraphBuilder.buildGraph(
           executionGraph,
           jobGraph,
@@ -1282,6 +1285,7 @@ class JobManager(
           jobMetrics,
           numSlots,
           blobServer,
+          Time.milliseconds(allocationTimeout),
           log.logger)
         
         if (registerNewGraph) {

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointSettingsSerializableTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointSettingsSerializableTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointSettingsSerializableTest.java
index 8abff4f..9456f10 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointSettingsSerializableTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointSettingsSerializableTest.java
@@ -29,11 +29,11 @@ import org.apache.flink.runtime.execution.Environment;
 import org.apache.flink.runtime.executiongraph.ExecutionGraph;
 import org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder;
 import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
-import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration;
 import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings;
+import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider;
 import org.apache.flink.runtime.query.TaskKvStateRegistry;
 import org.apache.flink.runtime.state.AbstractKeyedStateBackend;
 import org.apache.flink.runtime.state.CheckpointStorage;
@@ -41,7 +41,6 @@ import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation;
 import org.apache.flink.runtime.state.KeyGroupRange;
 import org.apache.flink.runtime.state.OperatorStateBackend;
 import org.apache.flink.runtime.state.StateBackend;
-import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.util.SerializedValue;
 import org.apache.flink.util.TestLogger;
@@ -95,6 +94,7 @@ public class CheckpointSettingsSerializableTest extends TestLogger {
 		// distributed execution
 		final JobGraph copy = CommonTestUtils.createCopySerializable(jobGraph);
 
+		final Time timeout = Time.seconds(10L);
 		final ExecutionGraph eg = ExecutionGraphBuilder.buildGraph(
 			null,
 			copy,
@@ -104,11 +104,12 @@ public class CheckpointSettingsSerializableTest extends TestLogger {
 			mock(SlotProvider.class),
 			classLoader,
 			new StandaloneCheckpointRecoveryFactory(),
-			Time.seconds(10),
+			timeout,
 			new NoRestartStrategy(),
 			new UnregisteredMetricsGroup(),
 			10,
 			VoidBlobWriter.getInstance(),
+			timeout,
 			log);
 
 		assertEquals(1, eg.getCheckpointCoordinator().getNumberOfRegisteredMasterHooks());

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
index a53d6d2..4b7449b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
@@ -78,16 +78,18 @@ public class ExecutionGraphCheckpointCoordinatorTest {
 	private ExecutionGraph createExecutionGraphAndEnableCheckpointing(
 			CheckpointIDCounter counter,
 			CompletedCheckpointStore store) throws Exception {
+		final Time timeout = Time.days(1L);
 		ExecutionGraph executionGraph = new ExecutionGraph(
 			new DummyJobInformation(),
 			TestingUtils.defaultExecutor(),
 			TestingUtils.defaultExecutor(),
-			Time.days(1L),
+			timeout,
 			new NoRestartStrategy(),
 			new RestartAllStrategy.Factory(),
 			new Scheduler(TestingUtils.defaultExecutionContext()),
 			ClassLoader.getSystemClassLoader(),
-			VoidBlobWriter.getInstance());
+			VoidBlobWriter.getInstance(),
+			timeout);
 
 		executionGraph.enableCheckpointing(
 				100,

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
index 55166d5..12b4277 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
@@ -166,7 +166,8 @@ public class ExecutionGraphDeploymentTest extends TestLogger {
 				new RestartAllStrategy.Factory(),
 				new Scheduler(TestingUtils.defaultExecutionContext()),
 				ExecutionGraph.class.getClassLoader(),
-				blobWriter);
+				blobWriter,
+				AkkaUtils.getDefaultTimeout());
 
 			checkJobOffloaded(eg);
 
@@ -448,7 +449,8 @@ public class ExecutionGraphDeploymentTest extends TestLogger {
 			new RestartAllStrategy.Factory(),
 			scheduler,
 			ExecutionGraph.class.getClassLoader(),
-			blobWriter);
+			blobWriter,
+			AkkaUtils.getDefaultTimeout());
 
 		checkJobOffloaded(eg);
 
@@ -529,7 +531,8 @@ public class ExecutionGraphDeploymentTest extends TestLogger {
 			new RestartAllStrategy.Factory(),
 			scheduler,
 			ExecutionGraph.class.getClassLoader(),
-			blobWriter);
+			blobWriter,
+			AkkaUtils.getDefaultTimeout());
 		checkJobOffloaded(eg);
 		
 		eg.setQueuedSchedulingAllowed(false);
@@ -694,6 +697,7 @@ public class ExecutionGraphDeploymentTest extends TestLogger {
 					false),
 				null));
 
+		final Time timeout = Time.seconds(10L);
 		return ExecutionGraphBuilder.buildGraph(
 			null,
 			jobGraph,
@@ -703,11 +707,12 @@ public class ExecutionGraphDeploymentTest extends TestLogger {
 			new ProgrammedSlotProvider(1),
 			getClass().getClassLoader(),
 			new StandaloneCheckpointRecoveryFactory(),
-			Time.seconds(10),
+			timeout,
 			new NoRestartStrategy(),
 			new UnregisteredMetricsGroup(),
 			1,
 			blobWriter,
+			timeout,
 			LoggerFactory.getLogger(getClass()));
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java
index caf89e8..18476a5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java
@@ -79,7 +79,7 @@ public class ExecutionGraphMetricsTest extends TestLogger {
 
 			CompletableFuture<LogicalSlot> slotFuture1 = CompletableFuture.completedFuture(new TestingLogicalSlot());
 			CompletableFuture<LogicalSlot> slotFuture2 = CompletableFuture.completedFuture(new TestingLogicalSlot());
-			when(scheduler.allocateSlot(any(ScheduledUnit.class), anyBoolean(), any(Collection.class))).thenReturn(slotFuture1, slotFuture2);
+			when(scheduler.allocateSlot(any(ScheduledUnit.class), anyBoolean(), any(Collection.class), any(Time.class))).thenReturn(slotFuture1, slotFuture2);
 
 			TestingRestartStrategy testingRestartStrategy = new TestingRestartStrategy();
 

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRescalingTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRescalingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRescalingTest.java
index f4a9f25..977ec14 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRescalingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRescalingTest.java
@@ -77,6 +77,7 @@ public class ExecutionGraphRescalingTest extends TestLogger {
 			new UnregisteredMetricsGroup(),
 			-1,
 			VoidBlobWriter.getInstance(),
+			AkkaUtils.getDefaultTimeout(),
 			TEST_LOGGER);
 
 		for (JobVertex jv : jobVertices) {
@@ -106,6 +107,7 @@ public class ExecutionGraphRescalingTest extends TestLogger {
 			new UnregisteredMetricsGroup(),
 			-1,
 			VoidBlobWriter.getInstance(),
+			AkkaUtils.getDefaultTimeout(),
 			TEST_LOGGER);
 
 		for (JobVertex jv : jobVertices) {
@@ -135,6 +137,7 @@ public class ExecutionGraphRescalingTest extends TestLogger {
 			new UnregisteredMetricsGroup(),
 			-1,
 			VoidBlobWriter.getInstance(),
+			AkkaUtils.getDefaultTimeout(),
 			TEST_LOGGER);
 
 		for (JobVertex jv : jobVertices) {
@@ -177,6 +180,7 @@ public class ExecutionGraphRescalingTest extends TestLogger {
 				new UnregisteredMetricsGroup(),
 				-1,
 				VoidBlobWriter.getInstance(),
+				AkkaUtils.getDefaultTimeout(),
 				TEST_LOGGER);
 
 			fail("Building the ExecutionGraph with a parallelism higher than the max parallelism should fail.");

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSchedulingTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSchedulingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSchedulingTest.java
index f75cb4b..b16aa96 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSchedulingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphSchedulingTest.java
@@ -28,19 +28,19 @@ import org.apache.flink.runtime.clusterframework.types.AllocationID;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
 import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
-import org.apache.flink.runtime.jobmaster.LogicalSlot;
 import org.apache.flink.runtime.instance.SimpleSlot;
-import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider;
+import org.apache.flink.runtime.instance.SimpleSlotContext;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
 import org.apache.flink.runtime.jobgraph.DistributionPattern;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobgraph.JobVertex;
 import org.apache.flink.runtime.jobgraph.ScheduleMode;
-import org.apache.flink.runtime.instance.SimpleSlotContext;
-import org.apache.flink.runtime.jobmaster.SlotOwner;
 import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway;
 import org.apache.flink.runtime.jobmanager.slots.TestingSlotOwner;
+import org.apache.flink.runtime.jobmaster.LogicalSlot;
+import org.apache.flink.runtime.jobmaster.SlotOwner;
+import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider;
 import org.apache.flink.runtime.messages.Acknowledge;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 import org.apache.flink.runtime.testtasks.NoOpInvokable;
@@ -57,9 +57,11 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
+import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.mock;
@@ -344,11 +346,11 @@ public class ExecutionGraphSchedulingTest extends TestLogger {
 	}
 
 	/**
-	 * This test verifies that the slot allocations times out after a certain time, and that
-	 * all slots are released in that case.
+	 * This tests makes sure that with eager scheduling no task is deployed if a single
+	 * slot allocation fails. Moreover we check that allocated slots will be returned.
 	 */
 	@Test
-	public void testTimeoutForSlotAllocation() throws Exception {
+	public void testEagerSchedulingWithSlotTimeout() throws Exception {
 
 		//  we construct a simple graph:    (task)
 
@@ -379,7 +381,7 @@ public class ExecutionGraphSchedulingTest extends TestLogger {
 		ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(parallelism);
 		slotProvider.addSlots(vertex.getID(), slotFutures);
 
-		final ExecutionGraph eg = createExecutionGraph(jobGraph, slotProvider, Time.milliseconds(20));
+		final ExecutionGraph eg = createExecutionGraph(jobGraph, slotProvider);
 
 		//  we complete one future
 		slotFutures[1].complete(slots[1]);
@@ -393,9 +395,13 @@ public class ExecutionGraphSchedulingTest extends TestLogger {
 		//  we complete another future
 		slotFutures[2].complete(slots[2]);
 
-		// since future[0] is still missing the while operation must time out
-		// we have no restarts allowed, so the job will go terminal
-		eg.getTerminationFuture().get(2000, TimeUnit.MILLISECONDS);
+		// check that the ExecutionGraph is not terminated yet
+		assertThat(eg.getTerminationFuture().isDone(), is(false));
+
+		// time out one of the slot futures
+		slotFutures[0].completeExceptionally(new TimeoutException("Test time out"));
+
+		assertThat(eg.getTerminationFuture().get(), is(JobStatus.FAILED));
 
 		// wait until all slots are back
 		for (int i = 0; i < parallelism - 1; i++) {
@@ -404,12 +410,6 @@ public class ExecutionGraphSchedulingTest extends TestLogger {
 
 		//  verify that no deployments have happened
 		verify(taskManager, times(0)).submitTask(any(TaskDeploymentDescriptor.class), any(Time.class));
-
-		for (CompletableFuture<LogicalSlot> future : slotFutures) {
-			if (future.isDone()) {
-				assertFalse(future.get().isAlive());
-			}
-		}
 	}
 
 	// ------------------------------------------------------------------------
@@ -435,6 +435,7 @@ public class ExecutionGraphSchedulingTest extends TestLogger {
 			new UnregisteredMetricsGroup(),
 			1,
 			VoidBlobWriter.getInstance(),
+			timeout,
 			log);
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
index 044cb52..a9c03a8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
@@ -354,6 +354,7 @@ public class ExecutionGraphTestUtils {
 			new UnregisteredMetricsGroup(),
 			1,
 			VoidBlobWriter.getInstance(),
+			timeout,
 			TEST_LOGGER);
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionTest.java
index 46dfd41..edae2c7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionTest.java
@@ -88,7 +88,8 @@ public class ExecutionTest extends TestLogger {
 		CompletableFuture<Execution> allocationFuture = execution.allocateAndAssignSlotForExecution(
 			slotProvider,
 			false,
-			LocationPreferenceConstraint.ALL);
+			LocationPreferenceConstraint.ALL,
+			TestingUtils.infiniteTime());
 
 		assertFalse(allocationFuture.isDone());
 
@@ -140,7 +141,8 @@ public class ExecutionTest extends TestLogger {
 		CompletableFuture<Execution> allocationFuture = execution.allocateAndAssignSlotForExecution(
 			slotProvider,
 			false,
-			LocationPreferenceConstraint.ALL);
+			LocationPreferenceConstraint.ALL,
+			TestingUtils.infiniteTime());
 
 		assertTrue(allocationFuture.isDone());
 
@@ -189,7 +191,8 @@ public class ExecutionTest extends TestLogger {
 		CompletableFuture<Execution> allocationFuture = execution.allocateAndAssignSlotForExecution(
 			slotProvider,
 			false,
-			LocationPreferenceConstraint.ALL);
+			LocationPreferenceConstraint.ALL,
+			TestingUtils.infiniteTime());
 
 		assertTrue(allocationFuture.isDone());
 

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexLocalityTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexLocalityTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexLocalityTest.java
index 14752e8..a207c7d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexLocalityTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexLocalityTest.java
@@ -23,23 +23,23 @@ import org.apache.flink.api.common.time.Time;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
 import org.apache.flink.runtime.blob.VoidBlobWriter;
-import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory;
 import org.apache.flink.runtime.checkpoint.JobManagerTaskRestore;
+import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory;
 import org.apache.flink.runtime.clusterframework.types.AllocationID;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.executiongraph.restart.FixedDelayRestartStrategy;
 import org.apache.flink.runtime.instance.SimpleSlot;
-import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider;
+import org.apache.flink.runtime.instance.SimpleSlotContext;
 import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
 import org.apache.flink.runtime.jobgraph.DistributionPattern;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.instance.SimpleSlotContext;
+import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway;
 import org.apache.flink.runtime.jobmaster.SlotContext;
 import org.apache.flink.runtime.jobmaster.SlotOwner;
-import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway;
+import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.testtasks.NoOpInvokable;
@@ -52,7 +52,6 @@ import java.lang.reflect.Field;
 import java.net.InetAddress;
 import java.util.Iterator;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -210,6 +209,7 @@ public class ExecutionVertexLocalityTest extends TestLogger {
 
 		JobGraph testJob = new JobGraph(jobId, "test job", source, target);
 
+		final Time timeout = Time.seconds(10L);
 		return ExecutionGraphBuilder.buildGraph(
 			null,
 			testJob,
@@ -219,11 +219,12 @@ public class ExecutionVertexLocalityTest extends TestLogger {
 			mock(SlotProvider.class),
 			getClass().getClassLoader(),
 			new StandaloneCheckpointRecoveryFactory(),
-			Time.of(10, TimeUnit.SECONDS),
+			timeout,
 			new FixedDelayRestartStrategy(10, 0L),
 			new UnregisteredMetricsGroup(),
 			1,
 			VoidBlobWriter.getInstance(),
+			timeout,
 			log);
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexSchedulingTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexSchedulingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexSchedulingTest.java
index 25e1207..2c49573 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexSchedulingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexSchedulingTest.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.runtime.executiongraph;
 
+import org.apache.flink.api.common.time.Time;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.instance.DummyActorGateway;
@@ -28,6 +29,7 @@ import org.apache.flink.runtime.jobmanager.scheduler.LocationPreferenceConstrain
 import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit;
 import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
 import org.apache.flink.runtime.jobmanager.slots.ActorTaskManagerGateway;
+import org.apache.flink.runtime.jobmaster.LogicalSlot;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 
 import org.junit.Test;
@@ -63,9 +65,9 @@ public class ExecutionVertexSchedulingTest {
 			assertTrue(slot.isReleased());
 
 			Scheduler scheduler = mock(Scheduler.class);
-			CompletableFuture<SimpleSlot> future = new CompletableFuture<>();
+			CompletableFuture<LogicalSlot> future = new CompletableFuture<>();
 			future.complete(slot);
-			when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean(), any(Collection.class))).thenReturn(future);
+			when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean(), any(Collection.class), any(Time.class))).thenReturn(future);
 
 			assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
 			// try to deploy to the slot
@@ -94,10 +96,10 @@ public class ExecutionVertexSchedulingTest {
 			slot.releaseSlot();
 			assertTrue(slot.isReleased());
 
-			final CompletableFuture<SimpleSlot> future = new CompletableFuture<>();
+			final CompletableFuture<LogicalSlot> future = new CompletableFuture<>();
 
 			Scheduler scheduler = mock(Scheduler.class);
-			when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean(), any(Collection.class))).thenReturn(future);
+			when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean(), any(Collection.class), any(Time.class))).thenReturn(future);
 
 			assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
 			// try to deploy to the slot
@@ -129,9 +131,9 @@ public class ExecutionVertexSchedulingTest {
 			final SimpleSlot slot = instance.allocateSimpleSlot();
 
 			Scheduler scheduler = mock(Scheduler.class);
-			CompletableFuture<SimpleSlot> future = new CompletableFuture<>();
+			CompletableFuture<LogicalSlot> future = new CompletableFuture<>();
 			future.complete(slot);
-			when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean(), any(Collection.class))).thenReturn(future);
+			when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean(), any(Collection.class), any(Time.class))).thenReturn(future);
 
 			assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
 

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PipelinedRegionFailoverConcurrencyTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PipelinedRegionFailoverConcurrencyTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PipelinedRegionFailoverConcurrencyTest.java
index 06647cc..65124ca 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PipelinedRegionFailoverConcurrencyTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/PipelinedRegionFailoverConcurrencyTest.java
@@ -310,16 +310,18 @@ public class PipelinedRegionFailoverConcurrencyTest extends TestLogger {
 			"test job");
 
 		// build a simple execution graph with on job vertex, parallelism 2
+		final Time timeout = Time.seconds(10L);
 		final ExecutionGraph graph = new ExecutionGraph(
 			jobInformation,
 			TestingUtils.defaultExecutor(),
 			TestingUtils.defaultExecutor(),
-			Time.seconds(10),
+			timeout,
 			restartStrategy,
 			failoverStrategy,
 			slotProvider,
 			getClass().getClassLoader(),
-			VoidBlobWriter.getInstance());
+			VoidBlobWriter.getInstance(),
+			timeout);
 
 		JobVertex jv = new JobVertex("test vertex");
 		jv.setInvokableClass(NoOpInvokable.class);

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ProgrammedSlotProvider.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ProgrammedSlotProvider.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ProgrammedSlotProvider.java
index f44626d..d93693e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ProgrammedSlotProvider.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ProgrammedSlotProvider.java
@@ -18,10 +18,11 @@
 
 package org.apache.flink.runtime.executiongraph;
 
-import org.apache.flink.runtime.jobmaster.LogicalSlot;
-import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider;
+import org.apache.flink.api.common.time.Time;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit;
+import org.apache.flink.runtime.jobmaster.LogicalSlot;
+import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 
 import java.util.Collection;
@@ -95,7 +96,8 @@ class ProgrammedSlotProvider implements SlotProvider {
 	public CompletableFuture<LogicalSlot> allocateSlot(
 			ScheduledUnit task,
 			boolean allowQueued,
-			Collection<TaskManagerLocation> preferredLocations) {
+			Collection<TaskManagerLocation> preferredLocations,
+			Time allocationTimeout) {
 		JobVertexID vertexId = task.getTaskToExecute().getVertex().getJobvertexId();
 		int subtask = task.getTaskToExecute().getParallelSubtaskIndex();
 

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RestartPipelinedRegionStrategyTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RestartPipelinedRegionStrategyTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RestartPipelinedRegionStrategyTest.java
index 059b8a9..9974574 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RestartPipelinedRegionStrategyTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RestartPipelinedRegionStrategyTest.java
@@ -103,7 +103,8 @@ public class RestartPipelinedRegionStrategyTest {
             new RestartPipelinedRegionStrategy.Factory(),
             scheduler,
             ExecutionGraph.class.getClassLoader(),
-			VoidBlobWriter.getInstance());
+			VoidBlobWriter.getInstance(),
+			AkkaUtils.getDefaultTimeout());
 		try {
 			eg.attachJobGraph(ordered);
 		}
@@ -187,7 +188,8 @@ public class RestartPipelinedRegionStrategyTest {
             new RestartPipelinedRegionStrategy.Factory(),
             scheduler,
             ExecutionGraph.class.getClassLoader(),
-			VoidBlobWriter.getInstance());
+			VoidBlobWriter.getInstance(),
+			AkkaUtils.getDefaultTimeout());
 		try {
 			eg.attachJobGraph(ordered);
 		}
@@ -276,7 +278,8 @@ public class RestartPipelinedRegionStrategyTest {
             new RestartPipelinedRegionStrategy.Factory(),
             scheduler,
             ExecutionGraph.class.getClassLoader(),
-			VoidBlobWriter.getInstance());
+			VoidBlobWriter.getInstance(),
+			AkkaUtils.getDefaultTimeout());
 		try {
 			eg.attachJobGraph(ordered);
 		}
@@ -356,7 +359,8 @@ public class RestartPipelinedRegionStrategyTest {
             new RestartPipelinedRegionStrategy.Factory(),
             scheduler,
             ExecutionGraph.class.getClassLoader(),
-			VoidBlobWriter.getInstance());
+			VoidBlobWriter.getInstance(),
+			AkkaUtils.getDefaultTimeout());
 		try {
 			eg.attachJobGraph(ordered);
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/PipelinedFailoverRegionBuildingTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/PipelinedFailoverRegionBuildingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/PipelinedFailoverRegionBuildingTest.java
index f94959d..05aed0e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/PipelinedFailoverRegionBuildingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/PipelinedFailoverRegionBuildingTest.java
@@ -627,6 +627,7 @@ public class PipelinedFailoverRegionBuildingTest extends TestLogger {
 				JobManagerOptions.EXECUTION_FAILOVER_STRATEGY,
 				FailoverStrategyLoader.PIPELINED_REGION_RESTART_STRATEGY_NAME);
 
+		final Time timeout = Time.seconds(10L);
 		return ExecutionGraphBuilder.buildGraph(
 			null,
 			jobGraph,
@@ -636,11 +637,12 @@ public class PipelinedFailoverRegionBuildingTest extends TestLogger {
 			mock(SlotProvider.class),
 			PipelinedFailoverRegionBuildingTest.class.getClassLoader(),
 			new StandaloneCheckpointRecoveryFactory(),
-			Time.seconds(10),
+			timeout,
 			new NoRestartStrategy(),
 			new UnregisteredMetricsGroup(),
 			1000,
 			VoidBlobWriter.getInstance(),
+			timeout,
 			log);
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java
index bffdab6..776ce16 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java
@@ -19,6 +19,7 @@
 package org.apache.flink.runtime.executiongraph.utils;
 
 import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
 import org.apache.flink.runtime.clusterframework.types.AllocationID;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.concurrent.FutureUtils;
@@ -74,7 +75,8 @@ public class SimpleSlotProvider implements SlotProvider, SlotOwner {
 	public CompletableFuture<LogicalSlot> allocateSlot(
 			ScheduledUnit task,
 			boolean allowQueued,
-			Collection<TaskManagerLocation> preferredLocations) {
+			Collection<TaskManagerLocation> preferredLocations,
+			Time allocationTimeout) {
 		final SlotContext slot;
 
 		synchronized (slots) {


[13/22] flink git commit: [FLINK-8212] [network] Pull EnvironmentInformation out of TaskManagerServices

Posted by tr...@apache.org.
[FLINK-8212] [network] Pull EnvironmentInformation out of TaskManagerServices

This closes #5458.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e3e601b3
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e3e601b3
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e3e601b3

Branch: refs/heads/master
Commit: e3e601b39492291b5837e58741bf7da46c76404a
Parents: 879185c
Author: zhangminglei <zm...@163.com>
Authored: Wed Feb 14 21:58:40 2018 +0800
Committer: Till Rohrmann <tr...@apache.org>
Committed: Thu Feb 15 08:44:20 2018 +0100

----------------------------------------------------------------------
 .../runtime/taskexecutor/TaskManagerRunner.java |  4 +-
 .../taskexecutor/TaskManagerServices.java       | 44 ++++++++++++--------
 .../minicluster/LocalFlinkMiniCluster.scala     |  4 +-
 .../flink/runtime/taskmanager/TaskManager.scala |  4 +-
 .../runtime/metrics/TaskManagerMetricsTest.java |  5 ++-
 .../NetworkBufferCalculationTest.java           | 34 +++------------
 6 files changed, 45 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/e3e601b3/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
index 3bf88d0..2fc796d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
@@ -285,7 +285,9 @@ public class TaskManagerRunner implements FatalErrorHandler {
 
 		TaskManagerServices taskManagerServices = TaskManagerServices.fromConfiguration(
 			taskManagerServicesConfiguration,
-			resourceID);
+			resourceID,
+			EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag(),
+			EnvironmentInformation.getMaxJvmHeapMemory());
 
 		TaskManagerMetricGroup taskManagerMetricGroup = MetricUtils.instantiateTaskManagerMetricGroup(
 			metricRegistry,

http://git-wip-us.apache.org/repos/asf/flink/blob/e3e601b3/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
index 0879481..33c8c28 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
@@ -46,7 +46,6 @@ import org.apache.flink.runtime.taskexecutor.slot.TaskSlotTable;
 import org.apache.flink.runtime.taskexecutor.slot.TimerService;
 import org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
-import org.apache.flink.runtime.util.EnvironmentInformation;
 import org.apache.flink.util.ExceptionUtils;
 import org.apache.flink.util.FlinkException;
 import org.apache.flink.util.Preconditions;
@@ -209,17 +208,21 @@ public class TaskManagerServices {
 	 *
 	 * @param resourceID resource ID of the task manager
 	 * @param taskManagerServicesConfiguration task manager configuration
+	 * @param freeHeapMemoryWithDefrag an estimate of the size of the free heap memory
+	 * @param maxJvmHeapMemory the maximum JVM heap size
 	 * @return task manager components
 	 * @throws Exception
 	 */
 	public static TaskManagerServices fromConfiguration(
 			TaskManagerServicesConfiguration taskManagerServicesConfiguration,
-			ResourceID resourceID) throws Exception {
+			ResourceID resourceID,
+			long freeHeapMemoryWithDefrag,
+			long maxJvmHeapMemory) throws Exception {
 
 		// pre-start checks
 		checkTempDirs(taskManagerServicesConfiguration.getTmpDirPaths());
 
-		final NetworkEnvironment network = createNetworkEnvironment(taskManagerServicesConfiguration);
+		final NetworkEnvironment network = createNetworkEnvironment(taskManagerServicesConfiguration, maxJvmHeapMemory);
 		network.start();
 
 		final TaskManagerLocation taskManagerLocation = new TaskManagerLocation(
@@ -228,7 +231,7 @@ public class TaskManagerServices {
 			network.getConnectionManager().getDataPort());
 
 		// this call has to happen strictly after the network stack has been initialized
-		final MemoryManager memoryManager = createMemoryManager(taskManagerServicesConfiguration);
+		final MemoryManager memoryManager = createMemoryManager(taskManagerServicesConfiguration, freeHeapMemoryWithDefrag, maxJvmHeapMemory);
 
 		// start the I/O manager, it will create some temp directories.
 		final IOManager ioManager = new IOManagerAsync(taskManagerServicesConfiguration.getTmpDirPaths());
@@ -270,10 +273,15 @@ public class TaskManagerServices {
 	 * Creates a {@link MemoryManager} from the given {@link TaskManagerServicesConfiguration}.
 	 *
 	 * @param taskManagerServicesConfiguration to create the memory manager from
+	 * @param freeHeapMemoryWithDefrag an estimate of the size of the free heap memory
+	 * @param maxJvmHeapMemory the maximum JVM heap size
 	 * @return Memory manager
 	 * @throws Exception
 	 */
-	private static MemoryManager createMemoryManager(TaskManagerServicesConfiguration taskManagerServicesConfiguration) throws Exception {
+	private static MemoryManager createMemoryManager(
+			TaskManagerServicesConfiguration taskManagerServicesConfiguration,
+			long freeHeapMemoryWithDefrag,
+			long maxJvmHeapMemory) throws Exception {
 		// computing the amount of memory to use depends on how much memory is available
 		// it strictly needs to happen AFTER the network stack has been initialized
 
@@ -299,7 +307,7 @@ public class TaskManagerServices {
 
 			if (memType == MemoryType.HEAP) {
 				// network buffers allocated off-heap -> use memoryFraction of the available heap:
-				long relativeMemSize = (long) (EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag() * memoryFraction);
+				long relativeMemSize = (long) (freeHeapMemoryWithDefrag * memoryFraction);
 				if (preAllocateMemory) {
 					LOG.info("Using {} of the currently free heap space for managed heap memory ({} MB)." ,
 						memoryFraction , relativeMemSize >> 20);
@@ -313,8 +321,7 @@ public class TaskManagerServices {
 				// calculateHeapSizeMB(long totalJavaMemorySizeMB, Configuration config)), i.e.
 				// maxJvmHeap = jvmTotalNoNet - jvmTotalNoNet * memoryFraction = jvmTotalNoNet * (1 - memoryFraction)
 				// directMemorySize = jvmTotalNoNet * memoryFraction
-				long maxJvmHeap = EnvironmentInformation.getMaxJvmHeapMemory();
-				long directMemorySize = (long) (maxJvmHeap / (1.0 - memoryFraction) * memoryFraction);
+				long directMemorySize = (long) (maxJvmHeapMemory / (1.0 - memoryFraction) * memoryFraction);
 				if (preAllocateMemory) {
 					LOG.info("Using {} of the maximum memory size for managed off-heap memory ({} MB)." ,
 						memoryFraction, directMemorySize >> 20);
@@ -356,15 +363,17 @@ public class TaskManagerServices {
 	 * Creates the {@link NetworkEnvironment} from the given {@link TaskManagerServicesConfiguration}.
 	 *
 	 * @param taskManagerServicesConfiguration to construct the network environment from
+	 * @param maxJvmHeapMemory the maximum JVM heap size
 	 * @return Network environment
 	 * @throws IOException
 	 */
 	private static NetworkEnvironment createNetworkEnvironment(
-			TaskManagerServicesConfiguration taskManagerServicesConfiguration) throws IOException {
+			TaskManagerServicesConfiguration taskManagerServicesConfiguration,
+			long maxJvmHeapMemory) {
 
 		NetworkEnvironmentConfiguration networkEnvironmentConfiguration = taskManagerServicesConfiguration.getNetworkConfig();
 
-		final long networkBuf = calculateNetworkBufferMemory(taskManagerServicesConfiguration);
+		final long networkBuf = calculateNetworkBufferMemory(taskManagerServicesConfiguration, maxJvmHeapMemory);
 		int segmentSize = networkEnvironmentConfiguration.networkBufferSize();
 
 		// tolerate offcuts between intended and allocated memory due to segmentation (will be available to the user-space memory)
@@ -530,10 +539,11 @@ public class TaskManagerServices {
 	 * </ul>.
 	 *
 	 * @param tmConfig task manager services configuration object
+	 * @param maxJvmHeapMemory the maximum JVM heap size
 	 *
 	 * @return memory to use for network buffers (in bytes)
 	 */
-	public static long calculateNetworkBufferMemory(TaskManagerServicesConfiguration tmConfig) {
+	public static long calculateNetworkBufferMemory(TaskManagerServicesConfiguration tmConfig, long maxJvmHeapMemory) {
 		final NetworkEnvironmentConfiguration networkConfig = tmConfig.getNetworkConfig();
 
 		final float networkBufFraction = networkConfig.networkBufFraction();
@@ -553,11 +563,9 @@ public class TaskManagerServices {
 
 		final MemoryType memType = tmConfig.getMemoryType();
 
-		final long maxMemory = EnvironmentInformation.getMaxJvmHeapMemory();
-
 		final long jvmHeapNoNet;
 		if (memType == MemoryType.HEAP) {
-			jvmHeapNoNet = maxMemory;
+			jvmHeapNoNet = maxJvmHeapMemory;
 		} else if (memType == MemoryType.OFF_HEAP) {
 
 			// check if a value has been configured
@@ -567,13 +575,13 @@ public class TaskManagerServices {
 				// The maximum heap memory has been adjusted according to configuredMemory, i.e.
 				// maxJvmHeap = jvmHeapNoNet - configuredMemory
 
-				jvmHeapNoNet = maxMemory + configuredMemory;
+				jvmHeapNoNet = maxJvmHeapMemory + configuredMemory;
 			} else {
 				// The maximum heap memory has been adjusted according to the fraction, i.e.
 				// maxJvmHeap = jvmHeapNoNet - jvmHeapNoNet * managedFraction = jvmHeapNoNet * (1 - managedFraction)
 
 				final float managedFraction = tmConfig.getMemoryFraction();
-				jvmHeapNoNet = (long) (maxMemory / (1.0 - managedFraction));
+				jvmHeapNoNet = (long) (maxJvmHeapMemory / (1.0 - managedFraction));
 			}
 		} else {
 			throw new RuntimeException("No supported memory type detected.");
@@ -586,13 +594,13 @@ public class TaskManagerServices {
 			(long) (jvmHeapNoNet / (1.0 - networkBufFraction) * networkBufFraction)));
 
 		TaskManagerServicesConfiguration
-			.checkConfigParameter(networkBufBytes < maxMemory,
+			.checkConfigParameter(networkBufBytes < maxJvmHeapMemory,
 				"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
 				"(" + TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
 					TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
 					TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
 				"Network buffer memory size too large: " + networkBufBytes + " >= " +
-					maxMemory + "(maximum JVM heap size)");
+					maxJvmHeapMemory + "(maximum JVM heap size)");
 
 		return networkBufBytes;
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/e3e601b3/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
index e864306..3173948 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
@@ -237,7 +237,9 @@ class LocalFlinkMiniCluster(
 
     val taskManagerServices = TaskManagerServices.fromConfiguration(
       taskManagerServicesConfiguration,
-      resourceID)
+      resourceID,
+      EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag,
+      EnvironmentInformation.getMaxJvmHeapMemory)
 
     val taskManagerMetricGroup = MetricUtils.instantiateTaskManagerMetricGroup(
       metricRegistryOpt.get,

http://git-wip-us.apache.org/repos/asf/flink/blob/e3e601b3/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 8dbb01d..54549ef 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -2012,7 +2012,9 @@ object TaskManager {
 
     val taskManagerServices = TaskManagerServices.fromConfiguration(
       taskManagerServicesConfiguration,
-      resourceID)
+      resourceID,
+      EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag,
+      EnvironmentInformation.getMaxJvmHeapMemory)
 
     val taskManagerMetricGroup = MetricUtils.instantiateTaskManagerMetricGroup(
       metricRegistry,

http://git-wip-us.apache.org/repos/asf/flink/blob/e3e601b3/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/TaskManagerMetricsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/TaskManagerMetricsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/TaskManagerMetricsTest.java
index eec7165..fe22095 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/TaskManagerMetricsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/TaskManagerMetricsTest.java
@@ -33,6 +33,7 @@ import org.apache.flink.runtime.taskexecutor.TaskManagerServices;
 import org.apache.flink.runtime.taskexecutor.TaskManagerServicesConfiguration;
 import org.apache.flink.runtime.taskmanager.TaskManager;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.util.EnvironmentInformation;
 import org.apache.flink.util.TestLogger;
 
 import akka.actor.ActorRef;
@@ -96,7 +97,9 @@ public class TaskManagerMetricsTest extends TestLogger {
 
 			TaskManagerServices taskManagerServices = TaskManagerServices.fromConfiguration(
 				taskManagerServicesConfiguration,
-				tmResourceID);
+				tmResourceID,
+				EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag(),
+				EnvironmentInformation.getMaxJvmHeapMemory());
 
 			TaskManagerMetricGroup taskManagerMetricGroup = MetricUtils.instantiateTaskManagerMetricGroup(
 				metricRegistry,

http://git-wip-us.apache.org/repos/asf/flink/blob/e3e601b3/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NetworkBufferCalculationTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NetworkBufferCalculationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NetworkBufferCalculationTest.java
index 6c145e8..bc6f238 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NetworkBufferCalculationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NetworkBufferCalculationTest.java
@@ -21,73 +21,51 @@ package org.apache.flink.runtime.taskexecutor;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.core.memory.MemoryType;
 import org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration;
-import org.apache.flink.runtime.util.EnvironmentInformation;
 import org.apache.flink.testutils.category.OldAndFlip6;
 import org.apache.flink.util.TestLogger;
 
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
 
 import java.net.InetAddress;
 
 import static org.junit.Assert.assertEquals;
-import static org.powermock.api.mockito.PowerMockito.when;
 
 /**
- * Tests the network buffer calculation from heap size which requires mocking
- * to have control over the {@link EnvironmentInformation}.
- *
- *
- * <p>This should be refactored once we have pulled {@link EnvironmentInformation} out of
- * {@link TaskManagerServices}. See FLINK-8212 for more information.
+ * Tests the network buffer calculation from heap size.
  */
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(EnvironmentInformation.class)
 @Category(OldAndFlip6.class)
 public class NetworkBufferCalculationTest extends TestLogger {
 
 	/**
-	 * Test for {@link TaskManagerServices#calculateNetworkBufferMemory(TaskManagerServicesConfiguration)}
+	 * Test for {@link TaskManagerServices#calculateNetworkBufferMemory(TaskManagerServicesConfiguration, long)}
 	 * using the same (manual) test cases as in {@link TaskManagerServicesTest#calculateHeapSizeMB()}.
 	 */
 	@Test
 	public void calculateNetworkBufFromHeapSize() throws Exception {
-		PowerMockito.mockStatic(EnvironmentInformation.class);
-		// some defaults:
-		when(EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag()).thenReturn(1000L << 20); // 1000MB
-		when(EnvironmentInformation.getMaxJvmHeapMemory()).thenReturn(1000L << 20); // 1000MB
-
 		TaskManagerServicesConfiguration tmConfig;
 
 		tmConfig = getTmConfig(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue(),
 			TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(),
 			0.1f, 60L << 20, 1L << 30, MemoryType.HEAP);
-		when(EnvironmentInformation.getMaxJvmHeapMemory()).thenReturn(900L << 20); // 900MB
 		assertEquals((100L << 20) + 1 /* one too many due to floating point imprecision */,
-			TaskManagerServices.calculateNetworkBufferMemory(tmConfig));
+			TaskManagerServices.calculateNetworkBufferMemory(tmConfig, 900L << 20)); // 900MB
 
 		tmConfig = getTmConfig(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue(),
 			TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(),
 			0.2f, 60L << 20, 1L << 30, MemoryType.HEAP);
-		when(EnvironmentInformation.getMaxJvmHeapMemory()).thenReturn(800L << 20); // 800MB
 		assertEquals((200L << 20) + 3 /* slightly too many due to floating point imprecision */,
-			TaskManagerServices.calculateNetworkBufferMemory(tmConfig));
+			TaskManagerServices.calculateNetworkBufferMemory(tmConfig, 800L << 20)); // 800MB
 
 		tmConfig = getTmConfig(10, TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(),
 			0.1f, 60L << 20, 1L << 30, MemoryType.OFF_HEAP);
-		when(EnvironmentInformation.getMaxJvmHeapMemory()).thenReturn(890L << 20); // 890MB
 		assertEquals((100L << 20) + 1 /* one too many due to floating point imprecision */,
-			TaskManagerServices.calculateNetworkBufferMemory(tmConfig));
+			TaskManagerServices.calculateNetworkBufferMemory(tmConfig, 890L << 20)); // 890MB
 
 		tmConfig = getTmConfig(-1, 0.1f,
 			0.1f, 60L << 20, 1L << 30, MemoryType.OFF_HEAP);
-		when(EnvironmentInformation.getMaxJvmHeapMemory()).thenReturn(810L << 20); // 810MB
 		assertEquals((100L << 20) + 1 /* one too many due to floating point imprecision */,
-			TaskManagerServices.calculateNetworkBufferMemory(tmConfig));
+			TaskManagerServices.calculateNetworkBufferMemory(tmConfig, 810L << 20)); // 810MB
 	}
 
 	/**


[14/22] flink git commit: [FLINK-7713][flip6] Implement JarUploadHandler

Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java
new file mode 100644
index 0000000..735b12d
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java
@@ -0,0 +1,541 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rest;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.RestOptions;
+import org.apache.flink.configuration.WebOptions;
+import org.apache.flink.runtime.rest.handler.AbstractRestHandler;
+import org.apache.flink.runtime.rest.handler.HandlerRequest;
+import org.apache.flink.runtime.rest.handler.RestHandlerException;
+import org.apache.flink.runtime.rest.handler.RestHandlerSpecification;
+import org.apache.flink.runtime.rest.messages.ConversionException;
+import org.apache.flink.runtime.rest.messages.EmptyMessageParameters;
+import org.apache.flink.runtime.rest.messages.EmptyResponseBody;
+import org.apache.flink.runtime.rest.messages.FileUpload;
+import org.apache.flink.runtime.rest.messages.MessageHeaders;
+import org.apache.flink.runtime.rest.messages.MessageParameters;
+import org.apache.flink.runtime.rest.messages.MessagePathParameter;
+import org.apache.flink.runtime.rest.messages.MessageQueryParameter;
+import org.apache.flink.runtime.rest.messages.RequestBody;
+import org.apache.flink.runtime.rest.messages.ResponseBody;
+import org.apache.flink.runtime.rest.util.RestClientException;
+import org.apache.flink.runtime.rpc.RpcUtils;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.webmonitor.RestfulGateway;
+import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
+import org.apache.flink.testutils.category.Flip6;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.TestLogger;
+
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandler;
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TemporaryFolder;
+
+import javax.annotation.Nonnull;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.net.HttpURLConnection;
+import java.net.InetSocketAddress;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * IT cases for {@link RestClient} and {@link RestServerEndpoint}.
+ */
+@Category(Flip6.class)
+public class RestServerEndpointITCase extends TestLogger {
+
+	private static final JobID PATH_JOB_ID = new JobID();
+	private static final JobID QUERY_JOB_ID = new JobID();
+	private static final String JOB_ID_KEY = "jobid";
+	private static final Time timeout = Time.seconds(10L);
+
+	@Rule
+	public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+	private RestServerEndpoint serverEndpoint;
+	private RestClient restClient;
+	private TestUploadHandler testUploadHandler;
+
+	@Before
+	public void setup() throws Exception {
+		Configuration config = new Configuration();
+		config.setInteger(RestOptions.REST_PORT, 0);
+		config.setString(WebOptions.UPLOAD_DIR, temporaryFolder.newFolder().getCanonicalPath());
+
+		RestServerEndpointConfiguration serverConfig = RestServerEndpointConfiguration.fromConfiguration(config);
+		RestClientConfiguration clientConfig = RestClientConfiguration.fromConfiguration(config);
+
+		final String restAddress = "http://localhost:1234";
+		RestfulGateway mockRestfulGateway = mock(RestfulGateway.class);
+		when(mockRestfulGateway.requestRestAddress(any(Time.class))).thenReturn(CompletableFuture.completedFuture(restAddress));
+		GatewayRetriever<RestfulGateway> mockGatewayRetriever = mock(GatewayRetriever.class);
+		when(mockGatewayRetriever.getNow()).thenReturn(Optional.of(mockRestfulGateway));
+
+		TestHandler testHandler = new TestHandler(
+			CompletableFuture.completedFuture(restAddress),
+			mockGatewayRetriever,
+			RpcUtils.INF_TIMEOUT);
+
+		testUploadHandler = new TestUploadHandler(
+			CompletableFuture.completedFuture(restAddress),
+			mockGatewayRetriever,
+			RpcUtils.INF_TIMEOUT);
+
+		serverEndpoint = new TestRestServerEndpoint(serverConfig, testHandler, testUploadHandler);
+		restClient = new TestRestClient(clientConfig);
+
+		serverEndpoint.start();
+	}
+
+	@After
+	public void teardown() {
+		if (restClient != null) {
+			restClient.shutdown(timeout);
+			restClient = null;
+		}
+
+		if (serverEndpoint != null) {
+			serverEndpoint.shutdown(timeout);
+			serverEndpoint = null;
+		}
+	}
+
+	/**
+	 * Tests that request are handled as individual units which don't interfere with each other.
+	 * This means that request responses can overtake each other.
+	 */
+	@Test
+	public void testRequestInterleaving() throws Exception {
+
+		TestParameters parameters = new TestParameters();
+		parameters.jobIDPathParameter.resolve(PATH_JOB_ID);
+		parameters.jobIDQueryParameter.resolve(Collections.singletonList(QUERY_JOB_ID));
+
+		// send first request and wait until the handler blocks
+		CompletableFuture<TestResponse> response1;
+		final InetSocketAddress serverAddress = serverEndpoint.getServerAddress();
+
+		synchronized (TestHandler.LOCK) {
+			response1 = restClient.sendRequest(
+				serverAddress.getHostName(),
+				serverAddress.getPort(),
+				new TestHeaders(),
+				parameters,
+				new TestRequest(1));
+			TestHandler.LOCK.wait();
+		}
+
+		// send second request and verify response
+		CompletableFuture<TestResponse> response2 = restClient.sendRequest(
+			serverAddress.getHostName(),
+			serverAddress.getPort(),
+			new TestHeaders(),
+			parameters,
+			new TestRequest(2));
+		assertEquals(2, response2.get().id);
+
+		// wake up blocked handler
+		synchronized (TestHandler.LOCK) {
+			TestHandler.LOCK.notifyAll();
+		}
+		// verify response to first request
+		assertEquals(1, response1.get().id);
+	}
+
+	/**
+	 * Tests that a bad handler request (HandlerRequest cannot be created) is reported as a BAD_REQUEST
+	 * and not an internal server error.
+	 *
+	 * <p>See FLINK-7663
+	 */
+	@Test
+	public void testBadHandlerRequest() throws Exception {
+		final InetSocketAddress serverAddress = serverEndpoint.getServerAddress();
+
+		final FaultyTestParameters parameters = new FaultyTestParameters();
+
+		parameters.faultyJobIDPathParameter.resolve(PATH_JOB_ID);
+		((TestParameters) parameters).jobIDQueryParameter.resolve(Collections.singletonList(QUERY_JOB_ID));
+
+		CompletableFuture<TestResponse> response = restClient.sendRequest(
+			serverAddress.getHostName(),
+			serverAddress.getPort(),
+			new TestHeaders(),
+			parameters,
+			new TestRequest(2));
+
+		try {
+			response.get();
+
+			Assert.fail("The request should fail with a bad request return code.");
+		} catch (ExecutionException ee) {
+			Throwable t = ExceptionUtils.stripExecutionException(ee);
+
+			Assert.assertTrue(t instanceof RestClientException);
+
+			RestClientException rce = (RestClientException) t;
+
+			assertEquals(HttpResponseStatus.BAD_REQUEST, rce.getHttpResponseStatus());
+		}
+	}
+
+	/**
+	 * Tests that multipart/form-data uploads work correctly.
+	 *
+	 * @see FileUploadHandler
+	 */
+	@Test
+	public void testFileUpload() throws Exception {
+		final String boundary = generateMultiPartBoundary();
+		final String crlf = "\r\n";
+		final String uploadedContent = "hello";
+		final HttpURLConnection connection = openHttpConnectionForUpload(boundary);
+
+		try (OutputStream output = connection.getOutputStream(); PrintWriter writer =
+			new PrintWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8), true)) {
+
+			writer.append("--" + boundary).append(crlf);
+			writer.append("Content-Disposition: form-data; name=\"foo\"; filename=\"bar\"").append(crlf);
+			writer.append("Content-Type: plain/text; charset=utf8").append(crlf);
+			writer.append(crlf).flush();
+			output.write(uploadedContent.getBytes(StandardCharsets.UTF_8));
+			output.flush();
+			writer.append(crlf).flush();
+			writer.append("--" + boundary + "--").append(crlf).flush();
+		}
+
+		assertEquals(200, connection.getResponseCode());
+		final Path lastUploadedPath = testUploadHandler.getLastUploadedPath();
+		assertEquals(uploadedContent, new String(Files.readAllBytes(lastUploadedPath), StandardCharsets.UTF_8));
+	}
+
+	/**
+	 * Sending multipart/form-data without a file should result in a bad request if the handler
+	 * expects a file upload.
+	 */
+	@Test
+	public void testMultiPartFormDataWithoutFileUpload() throws Exception {
+		final String boundary = generateMultiPartBoundary();
+		final String crlf = "\r\n";
+		final HttpURLConnection connection = openHttpConnectionForUpload(boundary);
+
+		try (OutputStream output = connection.getOutputStream(); PrintWriter writer =
+			new PrintWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8), true)) {
+
+			writer.append("--" + boundary).append(crlf);
+			writer.append("Content-Disposition: form-data; name=\"foo\"").append(crlf);
+			writer.append(crlf).flush();
+			output.write("test".getBytes(StandardCharsets.UTF_8));
+			output.flush();
+			writer.append(crlf).flush();
+			writer.append("--" + boundary + "--").append(crlf).flush();
+		}
+
+		assertEquals(400, connection.getResponseCode());
+	}
+
+	private HttpURLConnection openHttpConnectionForUpload(final String boundary) throws IOException {
+		final HttpURLConnection connection =
+			(HttpURLConnection) new URL(serverEndpoint.getRestAddress() + "/upload").openConnection();
+		connection.setDoOutput(true);
+		connection.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary);
+		return connection;
+	}
+
+	private static String generateMultiPartBoundary() {
+		return Long.toHexString(System.currentTimeMillis());
+	}
+
+	private static class TestRestServerEndpoint extends RestServerEndpoint {
+
+		private final TestHandler testHandler;
+
+		private final TestUploadHandler testUploadHandler;
+
+		TestRestServerEndpoint(
+			RestServerEndpointConfiguration configuration,
+			TestHandler testHandler,
+			TestUploadHandler testUploadHandler) {
+			super(configuration);
+
+			this.testHandler = Preconditions.checkNotNull(testHandler);
+			this.testUploadHandler = Preconditions.checkNotNull(testUploadHandler);
+		}
+
+		@Override
+		protected List<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> initializeHandlers(CompletableFuture<String> restAddressFuture) {
+			return Arrays.asList(
+				Tuple2.of(new TestHeaders(), testHandler),
+				Tuple2.of(TestUploadHeaders.INSTANCE, testUploadHandler));
+		}
+	}
+
+	private static class TestHandler extends AbstractRestHandler<RestfulGateway, TestRequest, TestResponse, TestParameters> {
+
+		public static final Object LOCK = new Object();
+
+		TestHandler(
+			CompletableFuture<String> localAddressFuture,
+			GatewayRetriever<RestfulGateway> leaderRetriever,
+			Time timeout) {
+			super(
+				localAddressFuture,
+				leaderRetriever,
+				timeout,
+				Collections.emptyMap(),
+				new TestHeaders());
+		}
+
+		@Override
+		protected CompletableFuture<TestResponse> handleRequest(@Nonnull HandlerRequest<TestRequest, TestParameters> request, RestfulGateway gateway) throws RestHandlerException {
+			assertEquals(request.getPathParameter(JobIDPathParameter.class), PATH_JOB_ID);
+			assertEquals(request.getQueryParameter(JobIDQueryParameter.class).get(0), QUERY_JOB_ID);
+
+			if (request.getRequestBody().id == 1) {
+				synchronized (LOCK) {
+					try {
+						LOCK.notifyAll();
+						LOCK.wait();
+					} catch (InterruptedException ignored) {
+					}
+				}
+			}
+			return CompletableFuture.completedFuture(new TestResponse(request.getRequestBody().id));
+		}
+	}
+
+	private static class TestRestClient extends RestClient {
+
+		TestRestClient(RestClientConfiguration configuration) {
+			super(configuration, TestingUtils.defaultExecutor());
+		}
+	}
+
+	private static class TestRequest implements RequestBody {
+		public final int id;
+
+		@JsonCreator
+		public TestRequest(@JsonProperty("id") int id) {
+			this.id = id;
+		}
+	}
+
+	private static class TestResponse implements ResponseBody {
+		public final int id;
+
+		@JsonCreator
+		public TestResponse(@JsonProperty("id") int id) {
+			this.id = id;
+		}
+	}
+
+	private static class TestHeaders implements MessageHeaders<TestRequest, TestResponse, TestParameters> {
+
+		@Override
+		public HttpMethodWrapper getHttpMethod() {
+			return HttpMethodWrapper.POST;
+		}
+
+		@Override
+		public String getTargetRestEndpointURL() {
+			return "/test/:jobid";
+		}
+
+		@Override
+		public Class<TestRequest> getRequestClass() {
+			return TestRequest.class;
+		}
+
+		@Override
+		public Class<TestResponse> getResponseClass() {
+			return TestResponse.class;
+		}
+
+		@Override
+		public HttpResponseStatus getResponseStatusCode() {
+			return HttpResponseStatus.OK;
+		}
+
+		@Override
+		public TestParameters getUnresolvedMessageParameters() {
+			return new TestParameters();
+		}
+	}
+
+	private static class TestParameters extends MessageParameters {
+		private final JobIDPathParameter jobIDPathParameter = new JobIDPathParameter();
+		private final JobIDQueryParameter jobIDQueryParameter = new JobIDQueryParameter();
+
+		@Override
+		public Collection<MessagePathParameter<?>> getPathParameters() {
+			return Collections.singleton(jobIDPathParameter);
+		}
+
+		@Override
+		public Collection<MessageQueryParameter<?>> getQueryParameters() {
+			return Collections.singleton(jobIDQueryParameter);
+		}
+	}
+
+	private static class FaultyTestParameters extends TestParameters {
+		private final FaultyJobIDPathParameter faultyJobIDPathParameter = new FaultyJobIDPathParameter();
+
+		@Override
+		public Collection<MessagePathParameter<?>> getPathParameters() {
+			return Collections.singleton(faultyJobIDPathParameter);
+		}
+	}
+
+	static class JobIDPathParameter extends MessagePathParameter<JobID> {
+		JobIDPathParameter() {
+			super(JOB_ID_KEY);
+		}
+
+		@Override
+		public JobID convertFromString(String value) {
+			return JobID.fromHexString(value);
+		}
+
+		@Override
+		protected String convertToString(JobID value) {
+			return value.toString();
+		}
+	}
+
+	static class FaultyJobIDPathParameter extends MessagePathParameter<JobID> {
+
+		FaultyJobIDPathParameter() {
+			super(JOB_ID_KEY);
+		}
+
+		@Override
+		protected JobID convertFromString(String value) throws ConversionException {
+			return JobID.fromHexString(value);
+		}
+
+		@Override
+		protected String convertToString(JobID value) {
+			return "foobar";
+		}
+	}
+
+	static class JobIDQueryParameter extends MessageQueryParameter<JobID> {
+		JobIDQueryParameter() {
+			super(JOB_ID_KEY, MessageParameterRequisiteness.MANDATORY);
+		}
+
+		@Override
+		public JobID convertValueFromString(String value) {
+			return JobID.fromHexString(value);
+		}
+
+		@Override
+		public String convertStringToValue(JobID value) {
+			return value.toString();
+		}
+	}
+
+	private static class TestUploadHandler extends AbstractRestHandler<RestfulGateway, FileUpload, EmptyResponseBody, EmptyMessageParameters> {
+
+		private volatile Path lastUploadedPath;
+
+		private TestUploadHandler(
+			final CompletableFuture<String> localRestAddress,
+			final GatewayRetriever<? extends RestfulGateway> leaderRetriever,
+			final Time timeout) {
+			super(localRestAddress, leaderRetriever, timeout, Collections.emptyMap(), TestUploadHeaders.INSTANCE);
+		}
+
+		@Override
+		protected CompletableFuture<EmptyResponseBody> handleRequest(@Nonnull final HandlerRequest<FileUpload, EmptyMessageParameters> request, @Nonnull final RestfulGateway gateway) throws RestHandlerException {
+			lastUploadedPath = request.getRequestBody().getPath();
+			return CompletableFuture.completedFuture(EmptyResponseBody.getInstance());
+		}
+
+		public Path getLastUploadedPath() {
+			return lastUploadedPath;
+		}
+	}
+
+	private enum TestUploadHeaders implements MessageHeaders<FileUpload, EmptyResponseBody, EmptyMessageParameters> {
+		INSTANCE;
+
+		@Override
+		public Class<EmptyResponseBody> getResponseClass() {
+			return EmptyResponseBody.class;
+		}
+
+		@Override
+		public HttpResponseStatus getResponseStatusCode() {
+			return HttpResponseStatus.OK;
+		}
+
+		@Override
+		public Class<FileUpload> getRequestClass() {
+			return FileUpload.class;
+		}
+
+		@Override
+		public EmptyMessageParameters getUnresolvedMessageParameters() {
+			return EmptyMessageParameters.getInstance();
+		}
+
+		@Override
+		public HttpMethodWrapper getHttpMethod() {
+			return HttpMethodWrapper.POST;
+		}
+
+		@Override
+		public String getTargetRestEndpointURL() {
+			return "/upload";
+		}
+	}
+}


[11/22] flink git commit: [FLINK-8608] [flip6] Implement MiniDispatcher for job mode

Posted by tr...@apache.org.
[FLINK-8608] [flip6] Implement MiniDispatcher for job mode

The MiniDispatcher is responsible for submitting the single job with which
a job mode cluster is started. Once the job has completed and if the cluster
has been started in detached mode, the MiniDispatcher will terminate.

In order to reduce code duplication, the MiniDispatcher is a sub class of the
Dispatcher which is started with a single job submitted job graph store.

This closes #5431.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/84418d0b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/84418d0b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/84418d0b

Branch: refs/heads/master
Commit: 84418d0b9183fbfeb3919fcb0d2f17708f786628
Parents: 2a15417
Author: Till Rohrmann <tr...@apache.org>
Authored: Wed Feb 14 15:44:06 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Thu Feb 15 08:44:19 2018 +0100

----------------------------------------------------------------------
 .../client/deployment/ClusterDescriptor.java    |   5 +-
 .../Flip6StandaloneClusterDescriptor.java       |   5 +-
 .../deployment/StandaloneClusterDescriptor.java |   5 +-
 .../client/cli/util/DummyClusterDescriptor.java |   5 +-
 .../entrypoint/MesosJobClusterEntrypoint.java   |  16 +-
 .../MesosSessionClusterEntrypoint.java          |  16 +-
 .../flink/runtime/dispatcher/Dispatcher.java    | 112 ++++--
 .../MemoryArchivedExecutionGraphStore.java      |  92 +++++
 .../runtime/dispatcher/MiniDispatcher.java      | 120 ++++++
 .../SingleJobSubmittedJobGraphStore.java        |  77 ++++
 .../dispatcher/StandaloneDispatcher.java        |  36 +-
 .../runtime/entrypoint/ClusterEntrypoint.java   | 383 +++++++++++++++++--
 .../entrypoint/JobClusterEntrypoint.java        | 301 ++-------------
 .../entrypoint/SessionClusterEntrypoint.java    | 202 +---------
 .../jobmaster/JobMasterRestEndpoint.java        |  62 ---
 .../jobmaster/MiniDispatcherRestEndpoint.java   |  64 ++++
 .../rest/handler/job/BlobServerPortHandler.java |   2 +-
 .../rest/handler/job/JobSubmitHandler.java      |   2 +-
 .../legacy/files/StaticFileServerHandler.java   |   2 +-
 .../TaskExecutorRegistrationSuccess.java        |   2 +-
 .../taskexecutor/slot/TaskSlotTable.java        |   2 +-
 .../runtime/webmonitor/WebMonitorEndpoint.java  |   6 +-
 .../runtime/webmonitor/WebMonitorUtils.java     |   2 +-
 .../runtime/dispatcher/DispatcherTest.java      |  54 +--
 .../MemoryArchivedExecutionGraphStore.java      |  92 -----
 .../runtime/dispatcher/MiniDispatcherTest.java  | 211 ++++++++++
 .../yarn/TestingYarnClusterDescriptor.java      |   5 +-
 .../java/org/apache/flink/yarn/YARNITCase.java  |   5 +-
 .../yarn/AbstractYarnClusterDescriptor.java     |  51 +--
 .../flink/yarn/Flip6YarnClusterDescriptor.java  |  24 ++
 .../flink/yarn/YarnClusterDescriptor.java       |   5 +-
 .../flink/yarn/AbstractYarnClusterTest.java     |   8 +
 32 files changed, 1173 insertions(+), 801 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-clients/src/main/java/org/apache/flink/client/deployment/ClusterDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/deployment/ClusterDescriptor.java b/flink-clients/src/main/java/org/apache/flink/client/deployment/ClusterDescriptor.java
index aadb5fe..f9f5d4b 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/deployment/ClusterDescriptor.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/deployment/ClusterDescriptor.java
@@ -56,12 +56,15 @@ public interface ClusterDescriptor<T> extends AutoCloseable {
 	 *
 	 * @param clusterSpecification Initial cluster specification with which the Flink cluster is launched
 	 * @param jobGraph JobGraph with which the job cluster is started
+	 * @param detached true if the cluster should be stopped after the job completion without serving the result,
+	 *                 otherwise false
 	 * @return Cluster client to talk to the Flink cluster
 	 * @throws ClusterDeploymentException if the cluster could not be deployed
 	 */
 	ClusterClient<T> deployJobCluster(
 		final ClusterSpecification clusterSpecification,
-		final JobGraph jobGraph) throws ClusterDeploymentException;
+		final JobGraph jobGraph,
+		final boolean detached) throws ClusterDeploymentException;
 
 	/**
 	 * Terminates the cluster with the given cluster id.

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-clients/src/main/java/org/apache/flink/client/deployment/Flip6StandaloneClusterDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/deployment/Flip6StandaloneClusterDescriptor.java b/flink-clients/src/main/java/org/apache/flink/client/deployment/Flip6StandaloneClusterDescriptor.java
index 8096da8..0a3286a 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/deployment/Flip6StandaloneClusterDescriptor.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/deployment/Flip6StandaloneClusterDescriptor.java
@@ -58,7 +58,10 @@ public class Flip6StandaloneClusterDescriptor implements ClusterDescriptor<Stand
 	}
 
 	@Override
-	public RestClusterClient<StandaloneClusterId> deployJobCluster(ClusterSpecification clusterSpecification, JobGraph jobGraph) {
+	public RestClusterClient<StandaloneClusterId> deployJobCluster(
+			ClusterSpecification clusterSpecification,
+			JobGraph jobGraph,
+			boolean detached) {
 		throw new UnsupportedOperationException("Can't deploy a standalone FLIP-6 per-job cluster.");
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-clients/src/main/java/org/apache/flink/client/deployment/StandaloneClusterDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/deployment/StandaloneClusterDescriptor.java b/flink-clients/src/main/java/org/apache/flink/client/deployment/StandaloneClusterDescriptor.java
index 62908fe..83bb136 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/deployment/StandaloneClusterDescriptor.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/deployment/StandaloneClusterDescriptor.java
@@ -57,7 +57,10 @@ public class StandaloneClusterDescriptor implements ClusterDescriptor<Standalone
 	}
 
 	@Override
-	public StandaloneClusterClient deployJobCluster(ClusterSpecification clusterSpecification, JobGraph jobGraph) {
+	public StandaloneClusterClient deployJobCluster(
+		ClusterSpecification clusterSpecification,
+		JobGraph jobGraph,
+		boolean detached) {
 		throw new UnsupportedOperationException("Can't deploy a standalone per-job cluster.");
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-clients/src/test/java/org/apache/flink/client/cli/util/DummyClusterDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/util/DummyClusterDescriptor.java b/flink-clients/src/test/java/org/apache/flink/client/cli/util/DummyClusterDescriptor.java
index 0ceb95e..df2f3f7 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/cli/util/DummyClusterDescriptor.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/cli/util/DummyClusterDescriptor.java
@@ -52,7 +52,10 @@ public class DummyClusterDescriptor<T> implements ClusterDescriptor<T> {
 	}
 
 	@Override
-	public ClusterClient<T> deployJobCluster(ClusterSpecification clusterSpecification, JobGraph jobGraph) {
+	public ClusterClient<T> deployJobCluster(
+			ClusterSpecification clusterSpecification,
+			JobGraph jobGraph,
+			boolean detached) {
 		return clusterClient;
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosJobClusterEntrypoint.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosJobClusterEntrypoint.java b/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosJobClusterEntrypoint.java
index 21864dc..b0054bb 100755
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosJobClusterEntrypoint.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosJobClusterEntrypoint.java
@@ -25,7 +25,6 @@ import org.apache.flink.mesos.runtime.clusterframework.MesosTaskManagerParameter
 import org.apache.flink.mesos.runtime.clusterframework.services.MesosServices;
 import org.apache.flink.mesos.runtime.clusterframework.services.MesosServicesUtils;
 import org.apache.flink.mesos.util.MesosConfiguration;
-import org.apache.flink.runtime.blob.BlobServer;
 import org.apache.flink.runtime.clusterframework.BootstrapTools;
 import org.apache.flink.runtime.clusterframework.ContainerSpecification;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
@@ -114,17 +113,6 @@ public class MesosJobClusterEntrypoint extends JobClusterEntrypoint {
 	}
 
 	@Override
-	protected void startClusterComponents(
-			Configuration configuration,
-			RpcService rpcService,
-			HighAvailabilityServices highAvailabilityServices,
-			BlobServer blobServer,
-			HeartbeatServices heartbeatServices,
-			MetricRegistry metricRegistry) throws Exception {
-		super.startClusterComponents(configuration, rpcService, highAvailabilityServices, blobServer, heartbeatServices, metricRegistry);
-	}
-
-	@Override
 	protected ResourceManager<?> createResourceManager(
 			Configuration configuration,
 			ResourceID resourceId,
@@ -179,11 +167,11 @@ public class MesosJobClusterEntrypoint extends JobClusterEntrypoint {
 	}
 
 	@Override
-	protected void stopClusterComponents(boolean cleanupHaData) throws Exception {
+	protected void stopClusterServices(boolean cleanupHaData) throws FlinkException {
 		Throwable exception = null;
 
 		try {
-			super.stopClusterComponents(cleanupHaData);
+			super.stopClusterServices(cleanupHaData);
 		} catch (Throwable t) {
 			exception = ExceptionUtils.firstOrSuppressed(t, exception);
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosSessionClusterEntrypoint.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosSessionClusterEntrypoint.java b/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosSessionClusterEntrypoint.java
index 68cf7c6..306d5b2 100755
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosSessionClusterEntrypoint.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosSessionClusterEntrypoint.java
@@ -25,7 +25,6 @@ import org.apache.flink.mesos.runtime.clusterframework.MesosTaskManagerParameter
 import org.apache.flink.mesos.runtime.clusterframework.services.MesosServices;
 import org.apache.flink.mesos.runtime.clusterframework.services.MesosServicesUtils;
 import org.apache.flink.mesos.util.MesosConfiguration;
-import org.apache.flink.runtime.blob.BlobServer;
 import org.apache.flink.runtime.clusterframework.BootstrapTools;
 import org.apache.flink.runtime.clusterframework.ContainerSpecification;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
@@ -104,17 +103,6 @@ public class MesosSessionClusterEntrypoint extends SessionClusterEntrypoint {
 	}
 
 	@Override
-	protected void startClusterComponents(
-			Configuration configuration,
-			RpcService rpcService,
-			HighAvailabilityServices highAvailabilityServices,
-			BlobServer blobServer,
-			HeartbeatServices heartbeatServices,
-			MetricRegistry metricRegistry) throws Exception {
-		super.startClusterComponents(configuration, rpcService, highAvailabilityServices, blobServer, heartbeatServices, metricRegistry);
-	}
-
-	@Override
 	protected ResourceManager<?> createResourceManager(
 			Configuration configuration,
 			ResourceID resourceId,
@@ -153,11 +141,11 @@ public class MesosSessionClusterEntrypoint extends SessionClusterEntrypoint {
 	}
 
 	@Override
-	protected void stopClusterComponents(boolean cleanupHaData) throws Exception {
+	protected void stopClusterServices(boolean cleanupHaData) throws FlinkException {
 		Throwable exception = null;
 
 		try {
-			super.stopClusterComponents(cleanupHaData);
+			super.stopClusterServices(cleanupHaData);
 		} catch (Throwable t) {
 			exception = t;
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
index c83dce0..92ccc9b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
@@ -102,19 +102,23 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 
 	private final ArchivedExecutionGraphStore archivedExecutionGraphStore;
 
+	private final JobManagerRunnerFactory jobManagerRunnerFactory;
+
 	@Nullable
 	protected final String restAddress;
 
-	protected Dispatcher(
+	public Dispatcher(
 			RpcService rpcService,
 			String endpointId,
 			Configuration configuration,
 			HighAvailabilityServices highAvailabilityServices,
+			SubmittedJobGraphStore submittedJobGraphStore,
 			ResourceManagerGateway resourceManagerGateway,
 			BlobServer blobServer,
 			HeartbeatServices heartbeatServices,
 			MetricRegistry metricRegistry,
 			ArchivedExecutionGraphStore archivedExecutionGraphStore,
+			JobManagerRunnerFactory jobManagerRunnerFactory,
 			FatalErrorHandler fatalErrorHandler,
 			@Nullable String restAddress) throws Exception {
 		super(rpcService, endpointId);
@@ -126,12 +130,12 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 		this.blobServer = Preconditions.checkNotNull(blobServer);
 		this.metricRegistry = Preconditions.checkNotNull(metricRegistry);
 		this.fatalErrorHandler = Preconditions.checkNotNull(fatalErrorHandler);
+		this.submittedJobGraphStore = Preconditions.checkNotNull(submittedJobGraphStore);
 
 		this.jobManagerSharedServices = JobManagerSharedServices.fromConfiguration(
 			configuration,
 			this.blobServer);
 
-		this.submittedJobGraphStore = highAvailabilityServices.getSubmittedJobGraphStore();
 		this.runningJobsRegistry = highAvailabilityServices.getRunningJobsRegistry();
 
 		jobManagerRunners = new HashMap<>(16);
@@ -141,6 +145,8 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 		this.restAddress = restAddress;
 
 		this.archivedExecutionGraphStore = Preconditions.checkNotNull(archivedExecutionGraphStore);
+
+		this.jobManagerRunnerFactory = Preconditions.checkNotNull(jobManagerRunnerFactory);
 	}
 
 	//------------------------------------------------------
@@ -226,7 +232,7 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 			final JobManagerRunner jobManagerRunner;
 
 			try {
-				jobManagerRunner = createJobManagerRunner(
+				jobManagerRunner = jobManagerRunnerFactory.createJobManagerRunner(
 					ResourceID.generate(),
 					jobGraph,
 					configuration,
@@ -237,7 +243,8 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 					jobManagerSharedServices,
 					metricRegistry,
 					new DispatcherOnCompleteActions(jobGraph.getJobID()),
-					fatalErrorHandler);
+					fatalErrorHandler,
+					restAddress);
 
 				jobManagerRunner.start();
 			} catch (Exception e) {
@@ -507,16 +514,18 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 			});
 	}
 
-	private void onFatalError(Throwable throwable) {
+	protected void onFatalError(Throwable throwable) {
 		log.error("Fatal error occurred in dispatcher {}.", getAddress(), throwable);
+
 		fatalErrorHandler.onFatalError(throwable);
 	}
 
-	private void jobReachedGloballyTerminalState(ArchivedExecutionGraph archivedExecutionGraph) {
+	protected void jobReachedGloballyTerminalState(ArchivedExecutionGraph archivedExecutionGraph) {
 		Preconditions.checkArgument(
 			archivedExecutionGraph.getState().isGloballyTerminalState(),
-			"Job " + archivedExecutionGraph.getJobID() + " is in state " +
-				archivedExecutionGraph.getState() + " which is not globally terminal.");
+			"Job %s is in state %s which is not globally terminal.",
+			archivedExecutionGraph.getJobID(),
+			archivedExecutionGraph.getState());
 
 		try {
 			archivedExecutionGraphStore.put(archivedExecutionGraph);
@@ -537,18 +546,13 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 		}
 	}
 
-	protected abstract JobManagerRunner createJobManagerRunner(
-		ResourceID resourceId,
-		JobGraph jobGraph,
-		Configuration configuration,
-		RpcService rpcService,
-		HighAvailabilityServices highAvailabilityServices,
-		HeartbeatServices heartbeatServices,
-		BlobServer blobServer,
-		JobManagerSharedServices jobManagerSharedServices,
-		MetricRegistry metricRegistry,
-		OnCompletionActions onCompleteActions,
-		FatalErrorHandler fatalErrorHandler) throws Exception;
+	protected void jobFinishedByOther(JobID jobId) {
+		try {
+			removeJob(jobId, false);
+		} catch (Exception e) {
+			log.warn("Could not properly remove job {} from the dispatcher.", jobId, e);
+		}
+	}
 
 	//------------------------------------------------------
 	// Leader contender
@@ -671,13 +675,67 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 			log.info("Job {} was finished by other JobManager.", jobId);
 
 			runAsync(
-				() -> {
-					try {
-						removeJob(jobId, false);
-					} catch (Exception e) {
-						log.warn("Could not properly remove job {} from the dispatcher.", jobId, e);
-					}
-				});
+				() -> Dispatcher.this.jobFinishedByOther(jobId));
+		}
+	}
+
+	//------------------------------------------------------
+	// Factories
+	//------------------------------------------------------
+
+	/**
+	 * Factory for a {@link JobManagerRunner}.
+	 */
+	@FunctionalInterface
+	public interface JobManagerRunnerFactory {
+		JobManagerRunner createJobManagerRunner(
+			ResourceID resourceId,
+			JobGraph jobGraph,
+			Configuration configuration,
+			RpcService rpcService,
+			HighAvailabilityServices highAvailabilityServices,
+			HeartbeatServices heartbeatServices,
+			BlobServer blobServer,
+			JobManagerSharedServices jobManagerServices,
+			MetricRegistry metricRegistry,
+			OnCompletionActions onCompleteActions,
+			FatalErrorHandler fatalErrorHandler,
+			@Nullable String restAddress) throws Exception;
+	}
+
+	/**
+	 * Singleton default factory for {@link JobManagerRunner}.
+	 */
+	public enum DefaultJobManagerRunnerFactory implements JobManagerRunnerFactory {
+		INSTANCE;
+
+		@Override
+		public JobManagerRunner createJobManagerRunner(
+				ResourceID resourceId,
+				JobGraph jobGraph,
+				Configuration configuration,
+				RpcService rpcService,
+				HighAvailabilityServices highAvailabilityServices,
+				HeartbeatServices heartbeatServices,
+				BlobServer blobServer,
+				JobManagerSharedServices jobManagerServices,
+				MetricRegistry metricRegistry,
+				OnCompletionActions onCompleteActions,
+				FatalErrorHandler fatalErrorHandler,
+				@Nullable String restAddress) throws Exception {
+			return new JobManagerRunner(
+				resourceId,
+				jobGraph,
+				configuration,
+				rpcService,
+				highAvailabilityServices,
+				heartbeatServices,
+				blobServer,
+				jobManagerServices,
+				metricRegistry,
+				onCompleteActions,
+				fatalErrorHandler,
+				restAddress);
 		}
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MemoryArchivedExecutionGraphStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MemoryArchivedExecutionGraphStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MemoryArchivedExecutionGraphStore.java
new file mode 100644
index 0000000..bcb7df2
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MemoryArchivedExecutionGraphStore.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.dispatcher;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph;
+import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.messages.webmonitor.JobDetails;
+import org.apache.flink.runtime.messages.webmonitor.JobsOverview;
+import org.apache.flink.runtime.webmonitor.WebMonitorUtils;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * {@link ArchivedExecutionGraphStore} implementation which stores the {@link ArchivedExecutionGraph}
+ * in memory.
+ */
+public class MemoryArchivedExecutionGraphStore implements ArchivedExecutionGraphStore {
+
+	private final Map<JobID, ArchivedExecutionGraph> serializableExecutionGraphs = new HashMap<>(4);
+
+	@Override
+	public int size() {
+		return serializableExecutionGraphs.size();
+	}
+
+	@Nullable
+	@Override
+	public ArchivedExecutionGraph get(JobID jobId) {
+		return serializableExecutionGraphs.get(jobId);
+	}
+
+	@Override
+	public void put(ArchivedExecutionGraph serializableExecutionGraph) throws IOException {
+		serializableExecutionGraphs.put(serializableExecutionGraph.getJobID(), serializableExecutionGraph);
+	}
+
+	@Override
+	public JobsOverview getStoredJobsOverview() {
+		Collection<JobStatus> allJobStatus = serializableExecutionGraphs.values().stream()
+			.map(ArchivedExecutionGraph::getState)
+			.collect(Collectors.toList());
+
+		return JobsOverview.create(allJobStatus);
+	}
+
+	@Override
+	public Collection<JobDetails> getAvailableJobDetails() {
+		return serializableExecutionGraphs.values().stream()
+			.map(WebMonitorUtils::createDetailsForJob)
+			.collect(Collectors.toList());
+	}
+
+	@Nullable
+	@Override
+	public JobDetails getAvailableJobDetails(JobID jobId) {
+		final ArchivedExecutionGraph archivedExecutionGraph = serializableExecutionGraphs.get(jobId);
+
+		if (archivedExecutionGraph != null) {
+			return WebMonitorUtils.createDetailsForJob(archivedExecutionGraph);
+		} else {
+			return null;
+		}
+	}
+
+	@Override
+	public void close() throws IOException {
+		serializableExecutionGraphs.clear();
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MiniDispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MiniDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MiniDispatcher.java
new file mode 100644
index 0000000..26d357e
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MiniDispatcher.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.dispatcher;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.blob.BlobServer;
+import org.apache.flink.runtime.entrypoint.ClusterEntrypoint;
+import org.apache.flink.runtime.entrypoint.JobClusterEntrypoint;
+import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph;
+import org.apache.flink.runtime.heartbeat.HeartbeatServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.messages.Acknowledge;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
+import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.util.FlinkException;
+
+import javax.annotation.Nullable;
+
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * Mini Dispatcher which is instantiated as the dispatcher component by the {@link JobClusterEntrypoint}.
+ *
+ * <p>The mini dispatcher is initialized with a single {@link JobGraph} which it runs.
+ *
+ * <p>Depending on the {@link ClusterEntrypoint.ExecutionMode}, the mini dispatcher will directly
+ * terminate after job completion if its execution mode is {@link ClusterEntrypoint.ExecutionMode#DETACHED}.
+ */
+public class MiniDispatcher extends Dispatcher {
+
+	private final JobClusterEntrypoint.ExecutionMode executionMode;
+
+	public MiniDispatcher(
+			RpcService rpcService,
+			String endpointId,
+			Configuration configuration,
+			HighAvailabilityServices highAvailabilityServices,
+			ResourceManagerGateway resourceManagerGateway,
+			BlobServer blobServer,
+			HeartbeatServices heartbeatServices,
+			MetricRegistry metricRegistry,
+			ArchivedExecutionGraphStore archivedExecutionGraphStore,
+			JobManagerRunnerFactory jobManagerRunnerFactory,
+			FatalErrorHandler fatalErrorHandler,
+			@Nullable String restAddress,
+			JobGraph jobGraph,
+			JobClusterEntrypoint.ExecutionMode executionMode) throws Exception {
+		super(
+			rpcService,
+			endpointId,
+			configuration,
+			highAvailabilityServices,
+			new SingleJobSubmittedJobGraphStore(jobGraph),
+			resourceManagerGateway,
+			blobServer,
+			heartbeatServices,
+			metricRegistry,
+			archivedExecutionGraphStore,
+			jobManagerRunnerFactory,
+			fatalErrorHandler,
+			restAddress);
+
+		this.executionMode = executionMode;
+	}
+
+	@Override
+	public CompletableFuture<Acknowledge> submitJob(JobGraph jobGraph, Time timeout) {
+		final CompletableFuture<Acknowledge> acknowledgeCompletableFuture = super.submitJob(jobGraph, timeout);
+
+		acknowledgeCompletableFuture.whenComplete(
+			(Acknowledge ignored, Throwable throwable) -> {
+				if (throwable != null) {
+					onFatalError(new FlinkException(
+						"Failed to submit job " + jobGraph.getJobID() + " in job mode.",
+						throwable));
+				}
+			});
+
+		return acknowledgeCompletableFuture;
+	}
+
+	@Override
+	protected void jobReachedGloballyTerminalState(ArchivedExecutionGraph archivedExecutionGraph) {
+		super.jobReachedGloballyTerminalState(archivedExecutionGraph);
+
+		if (executionMode == ClusterEntrypoint.ExecutionMode.DETACHED) {
+			// shut down since we don't have to wait for the execution result retrieval
+			shutDown();
+		}
+	}
+
+	@Override
+	protected void jobFinishedByOther(JobID jobId) {
+		super.jobFinishedByOther(jobId);
+
+		// shut down since we have done our job
+		shutDown();
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/SingleJobSubmittedJobGraphStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/SingleJobSubmittedJobGraphStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/SingleJobSubmittedJobGraphStore.java
new file mode 100644
index 0000000..26d3abc
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/SingleJobSubmittedJobGraphStore.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.dispatcher;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobmanager.SubmittedJobGraph;
+import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.Preconditions;
+
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * {@link SubmittedJobGraphStore} implementation for a single job.
+ */
+public class SingleJobSubmittedJobGraphStore implements SubmittedJobGraphStore {
+
+	private final JobGraph jobGraph;
+
+	public SingleJobSubmittedJobGraphStore(JobGraph jobGraph) {
+		this.jobGraph = Preconditions.checkNotNull(jobGraph);
+	}
+
+	@Override
+	public void start(SubmittedJobGraphListener jobGraphListener) throws Exception {
+		// noop
+	}
+
+	@Override
+	public void stop() throws Exception {
+		// noop
+	}
+
+	@Override
+	public SubmittedJobGraph recoverJobGraph(JobID jobId) throws Exception {
+		if (jobGraph.getJobID().equals(jobId)) {
+			return new SubmittedJobGraph(jobGraph, null);
+		} else {
+			throw new FlinkException("Could not recover job graph " + jobId + '.');
+		}
+	}
+
+	@Override
+	public void putJobGraph(SubmittedJobGraph jobGraph) throws Exception {
+		if (!jobGraph.getJobId().equals(jobGraph.getJobId())) {
+			throw new FlinkException("Cannot put additional jobs into this submitted job graph store.");
+		}
+	}
+
+	@Override
+	public void removeJobGraph(JobID jobId) throws Exception {
+		// ignore
+	}
+
+	@Override
+	public Collection<JobID> getJobIds() throws Exception {
+		return Collections.singleton(jobGraph.getJobID());
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java
index 6592e98..a7d21f3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java
@@ -20,13 +20,9 @@ package org.apache.flink.runtime.dispatcher;
 
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.blob.BlobServer;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.heartbeat.HeartbeatServices;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobmanager.OnCompletionActions;
-import org.apache.flink.runtime.jobmaster.JobManagerRunner;
-import org.apache.flink.runtime.jobmaster.JobManagerSharedServices;
 import org.apache.flink.runtime.jobmaster.JobMaster;
 import org.apache.flink.runtime.metrics.MetricRegistry;
 import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
@@ -51,6 +47,7 @@ public class StandaloneDispatcher extends Dispatcher {
 			HeartbeatServices heartbeatServices,
 			MetricRegistry metricRegistry,
 			ArchivedExecutionGraphStore archivedExecutionGraphStore,
+			JobManagerRunnerFactory jobManagerRunnerFactory,
 			FatalErrorHandler fatalErrorHandler,
 			@Nullable String restAddress) throws Exception {
 		super(
@@ -58,40 +55,13 @@ public class StandaloneDispatcher extends Dispatcher {
 			endpointId,
 			configuration,
 			highAvailabilityServices,
+			highAvailabilityServices.getSubmittedJobGraphStore(),
 			resourceManagerGateway,
 			blobServer,
 			heartbeatServices,
 			metricRegistry,
 			archivedExecutionGraphStore,
-			fatalErrorHandler,
-			restAddress);
-	}
-
-	@Override
-	protected JobManagerRunner createJobManagerRunner(
-			ResourceID resourceId,
-			JobGraph jobGraph,
-			Configuration configuration,
-			RpcService rpcService,
-			HighAvailabilityServices highAvailabilityServices,
-			HeartbeatServices heartbeatServices,
-			BlobServer blobServer,
-			JobManagerSharedServices jobManagerSharedServices,
-			MetricRegistry metricRegistry,
-			OnCompletionActions onCompleteActions,
-			FatalErrorHandler fatalErrorHandler) throws Exception {
-		// create the standard job manager runner
-		return new JobManagerRunner(
-			resourceId,
-			jobGraph,
-			configuration,
-			rpcService,
-			highAvailabilityServices,
-			heartbeatServices,
-			blobServer,
-			jobManagerSharedServices,
-			metricRegistry,
-			onCompleteActions,
+			jobManagerRunnerFactory,
 			fatalErrorHandler,
 			restAddress);
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java
index dfb1b4c..584fcae 100755
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java
@@ -20,25 +20,48 @@ package org.apache.flink.runtime.entrypoint;
 
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.GlobalConfiguration;
 import org.apache.flink.configuration.JobManagerOptions;
+import org.apache.flink.configuration.WebOptions;
 import org.apache.flink.core.fs.FileSystem;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.blob.BlobServer;
+import org.apache.flink.runtime.blob.TransientBlobCache;
+import org.apache.flink.runtime.blob.TransientBlobService;
+import org.apache.flink.runtime.clusterframework.ApplicationStatus;
 import org.apache.flink.runtime.clusterframework.BootstrapTools;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.ScheduledExecutor;
+import org.apache.flink.runtime.dispatcher.ArchivedExecutionGraphStore;
+import org.apache.flink.runtime.dispatcher.Dispatcher;
+import org.apache.flink.runtime.dispatcher.DispatcherGateway;
+import org.apache.flink.runtime.dispatcher.DispatcherId;
+import org.apache.flink.runtime.dispatcher.MiniDispatcher;
 import org.apache.flink.runtime.heartbeat.HeartbeatServices;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
+import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.metrics.MetricRegistry;
 import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
 import org.apache.flink.runtime.metrics.MetricRegistryImpl;
+import org.apache.flink.runtime.resourcemanager.ResourceManager;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerId;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.rpc.RpcService;
 import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
 import org.apache.flink.runtime.security.SecurityConfiguration;
 import org.apache.flink.runtime.security.SecurityContext;
 import org.apache.flink.runtime.security.SecurityUtils;
+import org.apache.flink.runtime.webmonitor.WebMonitorEndpoint;
+import org.apache.flink.runtime.webmonitor.retriever.LeaderGatewayRetriever;
+import org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever;
+import org.apache.flink.runtime.webmonitor.retriever.impl.AkkaQueryServiceRetriever;
+import org.apache.flink.runtime.webmonitor.retriever.impl.RpcGatewayRetriever;
 import org.apache.flink.util.ExceptionUtils;
 import org.apache.flink.util.FlinkException;
 import org.apache.flink.util.Preconditions;
@@ -47,12 +70,15 @@ import akka.actor.ActorSystem;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
 import javax.annotation.concurrent.GuardedBy;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import scala.concurrent.duration.FiniteDuration;
 
@@ -63,6 +89,10 @@ import scala.concurrent.duration.FiniteDuration;
  */
 public abstract class ClusterEntrypoint implements FatalErrorHandler {
 
+	public static final ConfigOption<String> EXECUTION_MODE = ConfigOptions
+		.key("internal.cluster.execution-mode")
+		.defaultValue(ExecutionMode.NORMAL.toString());
+
 	protected static final Logger LOG = LoggerFactory.getLogger(ClusterEntrypoint.class);
 
 	protected static final int SUCCESS_RETURN_CODE = 0;
@@ -76,20 +106,43 @@ public abstract class ClusterEntrypoint implements FatalErrorHandler {
 
 	private final CompletableFuture<Boolean> terminationFuture;
 
+	private final AtomicBoolean isTerminating = new AtomicBoolean(false);
+
 	@GuardedBy("lock")
-	private MetricRegistryImpl metricRegistry = null;
+	private MetricRegistryImpl metricRegistry;
 
 	@GuardedBy("lock")
-	private HighAvailabilityServices haServices = null;
+	private HighAvailabilityServices haServices;
 
 	@GuardedBy("lock")
-	private BlobServer blobServer = null;
+	private BlobServer blobServer;
 
 	@GuardedBy("lock")
-	private HeartbeatServices heartbeatServices = null;
+	private HeartbeatServices heartbeatServices;
 
 	@GuardedBy("lock")
-	private RpcService commonRpcService = null;
+	private RpcService commonRpcService;
+
+	@GuardedBy("lock")
+	private ResourceManager<?> resourceManager;
+
+	@GuardedBy("lock")
+	private Dispatcher dispatcher;
+
+	@GuardedBy("lock")
+	private LeaderRetrievalService dispatcherLeaderRetrievalService;
+
+	@GuardedBy("lock")
+	private LeaderRetrievalService resourceManagerRetrievalService;
+
+	@GuardedBy("lock")
+	private WebMonitorEndpoint<?> webMonitorEndpoint;
+
+	@GuardedBy("lock")
+	private ArchivedExecutionGraphStore archivedExecutionGraphStore;
+
+	@GuardedBy("lock")
+	private TransientBlobCache transientBlobCache;
 
 	protected ClusterEntrypoint(Configuration configuration) {
 		this.configuration = Preconditions.checkNotNull(configuration);
@@ -108,24 +161,18 @@ public abstract class ClusterEntrypoint implements FatalErrorHandler {
 
 			SecurityContext securityContext = installSecurityContext(configuration);
 
-			securityContext.runSecured(new Callable<Void>() {
-				@Override
-				public Void call() throws Exception {
-					runCluster(configuration);
+			securityContext.runSecured((Callable<Void>) () -> {
+				runCluster(configuration);
 
-					return null;
-				}
+				return null;
 			});
 		} catch (Throwable t) {
 			LOG.error("Cluster initialization failed.", t);
 
-			try {
-				shutDown(false);
-			} catch (Throwable st) {
-				LOG.error("Could not properly shut down cluster entrypoint.", st);
-			}
-
-			System.exit(STARTUP_FAILURE_RETURN_CODE);
+			shutDownAndTerminate(
+				STARTUP_FAILURE_RETURN_CODE,
+				ApplicationStatus.FAILED,
+				false);
 		}
 	}
 
@@ -163,33 +210,134 @@ public abstract class ClusterEntrypoint implements FatalErrorHandler {
 				blobServer,
 				heartbeatServices,
 				metricRegistry);
+
+			// TODO: Make shutDownAndTerminate non blocking to not use the global executor
+			dispatcher.getTerminationFuture().whenCompleteAsync(
+				(Boolean success, Throwable throwable) -> {
+					if (throwable != null) {
+						LOG.info("Could not properly terminate the Dispatcher.", throwable);
+					}
+
+					shutDownAndTerminate(
+						SUCCESS_RETURN_CODE,
+						ApplicationStatus.SUCCEEDED,
+						true);
+				});
 		}
 	}
 
 	protected void initializeServices(Configuration configuration) throws Exception {
-		assert(Thread.holdsLock(lock));
 
 		LOG.info("Initializing cluster services.");
 
-		final String bindAddress = configuration.getString(JobManagerOptions.ADDRESS);
-		final String portRange = getRPCPortRange(configuration);
+		synchronized (lock) {
+			final String bindAddress = configuration.getString(JobManagerOptions.ADDRESS);
+			final String portRange = getRPCPortRange(configuration);
 
-		commonRpcService = createRpcService(configuration, bindAddress, portRange);
+			commonRpcService = createRpcService(configuration, bindAddress, portRange);
 
-		// update the configuration used to create the high availability services
-		configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress());
-		configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort());
+			// update the configuration used to create the high availability services
+			configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress());
+			configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort());
 
-		haServices = createHaServices(configuration, commonRpcService.getExecutor());
-		blobServer = new BlobServer(configuration, haServices.createBlobStore());
-		blobServer.start();
-		heartbeatServices = createHeartbeatServices(configuration);
-		metricRegistry = createMetricRegistry(configuration);
+			haServices = createHaServices(configuration, commonRpcService.getExecutor());
+			blobServer = new BlobServer(configuration, haServices.createBlobStore());
+			blobServer.start();
+			heartbeatServices = createHeartbeatServices(configuration);
+			metricRegistry = createMetricRegistry(configuration);
 
-		// TODO: This is a temporary hack until we have ported the MetricQueryService to the new RpcEndpoint
-		// start the MetricQueryService
-		final ActorSystem actorSystem = ((AkkaRpcService) commonRpcService).getActorSystem();
-		metricRegistry.startQueryService(actorSystem, null);
+			// TODO: This is a temporary hack until we have ported the MetricQueryService to the new RpcEndpoint
+			// start the MetricQueryService
+			final ActorSystem actorSystem = ((AkkaRpcService) commonRpcService).getActorSystem();
+			metricRegistry.startQueryService(actorSystem, null);
+		}
+	}
+
+	protected void startClusterComponents(
+			Configuration configuration,
+			RpcService rpcService,
+			HighAvailabilityServices highAvailabilityServices,
+			BlobServer blobServer,
+			HeartbeatServices heartbeatServices,
+			MetricRegistry metricRegistry) throws Exception {
+		synchronized (lock) {
+			archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, rpcService.getScheduledExecutor());
+
+			dispatcherLeaderRetrievalService = highAvailabilityServices.getDispatcherLeaderRetriever();
+
+			resourceManagerRetrievalService = highAvailabilityServices.getResourceManagerLeaderRetriever();
+
+			final ClusterInformation clusterInformation = new ClusterInformation(
+				rpcService.getAddress(),
+				blobServer.getPort());
+
+			transientBlobCache = new TransientBlobCache(
+				configuration,
+				new InetSocketAddress(
+					clusterInformation.getBlobServerHostname(),
+					clusterInformation.getBlobServerPort()));
+
+			LeaderGatewayRetriever<DispatcherGateway> dispatcherGatewayRetriever = new RpcGatewayRetriever<>(
+				rpcService,
+				DispatcherGateway.class,
+				DispatcherId::new,
+				10,
+				Time.milliseconds(50L));
+
+			LeaderGatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever = new RpcGatewayRetriever<>(
+				rpcService,
+				ResourceManagerGateway.class,
+				ResourceManagerId::new,
+				10,
+				Time.milliseconds(50L));
+
+			// TODO: Remove once we have ported the MetricFetcher to the RpcEndpoint
+			final ActorSystem actorSystem = ((AkkaRpcService) rpcService).getActorSystem();
+			final Time timeout = Time.milliseconds(configuration.getLong(WebOptions.TIMEOUT));
+
+			webMonitorEndpoint = createRestEndpoint(
+				configuration,
+				dispatcherGatewayRetriever,
+				resourceManagerGatewayRetriever,
+				transientBlobCache,
+				rpcService.getExecutor(),
+				new AkkaQueryServiceRetriever(actorSystem, timeout),
+				highAvailabilityServices.getWebMonitorLeaderElectionService());
+
+			LOG.debug("Starting Dispatcher REST endpoint.");
+			webMonitorEndpoint.start();
+
+			resourceManager = createResourceManager(
+				configuration,
+				ResourceID.generate(),
+				rpcService,
+				highAvailabilityServices,
+				heartbeatServices,
+				metricRegistry,
+				this,
+				clusterInformation,
+				webMonitorEndpoint.getRestAddress());
+
+			dispatcher = createDispatcher(
+				configuration,
+				rpcService,
+				highAvailabilityServices,
+				resourceManager.getSelfGateway(ResourceManagerGateway.class),
+				blobServer,
+				heartbeatServices,
+				metricRegistry,
+				archivedExecutionGraphStore,
+				this,
+				webMonitorEndpoint.getRestAddress());
+
+			LOG.debug("Starting ResourceManager.");
+			resourceManager.start();
+			resourceManagerRetrievalService.start(resourceManagerGatewayRetriever);
+
+			LOG.debug("Starting Dispatcher.");
+			dispatcher.start();
+			dispatcherLeaderRetrievalService.start(dispatcherGatewayRetriever);
+		}
 	}
 
 	/**
@@ -228,7 +376,7 @@ public abstract class ClusterEntrypoint implements FatalErrorHandler {
 		return new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(configuration));
 	}
 
-	protected void shutDown(boolean cleanupHaData) throws FlinkException {
+	private void shutDown(boolean cleanupHaData) throws FlinkException {
 		LOG.info("Stopping {}.", getClass().getSimpleName());
 
 		Throwable exception = null;
@@ -236,11 +384,29 @@ public abstract class ClusterEntrypoint implements FatalErrorHandler {
 		synchronized (lock) {
 
 			try {
-				stopClusterComponents(cleanupHaData);
+				stopClusterComponents();
+			} catch (Throwable t) {
+				exception = ExceptionUtils.firstOrSuppressed(t, exception);
+			}
+
+			try {
+				stopClusterServices(cleanupHaData);
 			} catch (Throwable t) {
 				exception = ExceptionUtils.firstOrSuppressed(t, exception);
 			}
 
+			terminationFuture.complete(true);
+		}
+
+		if (exception != null) {
+			throw new FlinkException("Could not properly shut down the cluster entrypoint.", exception);
+		}
+	}
+
+	protected void stopClusterServices(boolean cleanupHaData) throws FlinkException {
+		Throwable exception = null;
+
+		synchronized (lock) {
 			if (metricRegistry != null) {
 				try {
 					metricRegistry.shutdown();
@@ -276,8 +442,6 @@ public abstract class ClusterEntrypoint implements FatalErrorHandler {
 					exception = ExceptionUtils.firstOrSuppressed(t, exception);
 				}
 			}
-
-			terminationFuture.complete(true);
 		}
 
 		if (exception != null) {
@@ -285,6 +449,68 @@ public abstract class ClusterEntrypoint implements FatalErrorHandler {
 		}
 	}
 
+	protected void stopClusterComponents() throws Exception {
+		synchronized (lock) {
+			Throwable exception = null;
+
+			if (webMonitorEndpoint != null) {
+				webMonitorEndpoint.shutdown(Time.seconds(10L));
+			}
+
+			if (dispatcherLeaderRetrievalService != null) {
+				try {
+					dispatcherLeaderRetrievalService.stop();
+				} catch (Throwable t) {
+					exception = ExceptionUtils.firstOrSuppressed(t, exception);
+				}
+			}
+
+			if (dispatcher != null) {
+				try {
+					dispatcher.shutDown();
+				} catch (Throwable t) {
+					exception = ExceptionUtils.firstOrSuppressed(t, exception);
+				}
+			}
+
+			if (resourceManagerRetrievalService != null) {
+				try {
+					resourceManagerRetrievalService.stop();
+				} catch (Throwable t) {
+					exception = ExceptionUtils.firstOrSuppressed(t, exception);
+				}
+			}
+
+			if (resourceManager != null) {
+				try {
+					resourceManager.shutDown();
+				} catch (Throwable t) {
+					exception = ExceptionUtils.firstOrSuppressed(t, exception);
+				}
+			}
+
+			if (archivedExecutionGraphStore != null) {
+				try {
+					archivedExecutionGraphStore.close();
+				} catch (Throwable t) {
+					exception = ExceptionUtils.firstOrSuppressed(t, exception);
+				}
+			}
+
+			if (transientBlobCache != null) {
+				try {
+					transientBlobCache.close();
+				} catch (Throwable t) {
+					exception = ExceptionUtils.firstOrSuppressed(t, exception);
+				}
+			}
+
+			if (exception != null) {
+				throw new FlinkException("Could not properly shut down the session cluster entry point.", exception);
+			}
+		}
+	}
+
 	@Override
 	public void onFatalError(Throwable exception) {
 		LOG.error("Fatal error occurred in the cluster entrypoint.", exception);
@@ -292,16 +518,74 @@ public abstract class ClusterEntrypoint implements FatalErrorHandler {
 		System.exit(RUNTIME_FAILURE_RETURN_CODE);
 	}
 
-	protected abstract void startClusterComponents(
+	// --------------------------------------------------
+	// Internal methods
+	// --------------------------------------------------
+
+	private void shutDownAndTerminate(
+		int returnCode,
+		ApplicationStatus applicationStatus,
+		boolean cleanupHaData) {
+
+		LOG.info("Shut down and terminate {} with return code {} and application status {}.",
+			getClass().getSimpleName(),
+			returnCode,
+			applicationStatus);
+
+		if (isTerminating.compareAndSet(false, true)) {
+			try {
+				shutDown(cleanupHaData);
+			} catch (Throwable t) {
+				LOG.info("Could not properly shut down cluster entrypoint.", t);
+			}
+
+			System.exit(returnCode);
+		} else {
+			LOG.debug("Concurrent termination call detected. Ignoring termination call with return code {} and application status {}.",
+				returnCode,
+				applicationStatus);
+		}
+	}
+
+	// --------------------------------------------------
+	// Abstract methods
+	// --------------------------------------------------
+
+	protected abstract Dispatcher createDispatcher(
 		Configuration configuration,
 		RpcService rpcService,
 		HighAvailabilityServices highAvailabilityServices,
+		ResourceManagerGateway resourceManagerGateway,
 		BlobServer blobServer,
 		HeartbeatServices heartbeatServices,
-		MetricRegistry metricRegistry) throws Exception;
+		MetricRegistry metricRegistry,
+		ArchivedExecutionGraphStore archivedExecutionGraphStore,
+		FatalErrorHandler fatalErrorHandler,
+		@Nullable String restAddress) throws Exception;
 
-	protected void stopClusterComponents(boolean cleanupHaData) throws Exception {
-	}
+	protected abstract ResourceManager<?> createResourceManager(
+		Configuration configuration,
+		ResourceID resourceId,
+		RpcService rpcService,
+		HighAvailabilityServices highAvailabilityServices,
+		HeartbeatServices heartbeatServices,
+		MetricRegistry metricRegistry,
+		FatalErrorHandler fatalErrorHandler,
+		ClusterInformation clusterInformation,
+		@Nullable String webInterfaceUrl) throws Exception;
+
+	protected abstract WebMonitorEndpoint<?> createRestEndpoint(
+		Configuration configuration,
+		LeaderGatewayRetriever<DispatcherGateway> dispatcherGatewayRetriever,
+		LeaderGatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
+		TransientBlobService transientBlobService,
+		Executor executor,
+		MetricQueryServiceRetriever metricQueryServiceRetriever,
+		LeaderElectionService leaderElectionService) throws Exception;
+
+	protected abstract ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
+		Configuration configuration,
+		ScheduledExecutor scheduledExecutor) throws IOException;
 
 	protected static ClusterConfiguration parseArguments(String[] args) {
 		ParameterTool parameterTool = ParameterTool.fromArgs(args);
@@ -314,4 +598,19 @@ public abstract class ClusterEntrypoint implements FatalErrorHandler {
 	protected static Configuration loadConfiguration(ClusterConfiguration clusterConfiguration) {
 		return GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir());
 	}
+
+	/**
+	 * Execution mode of the {@link MiniDispatcher}.
+	 */
+	public enum ExecutionMode {
+		/**
+		 * Waits until the job result has been served.
+		 */
+		NORMAL,
+
+		/**
+		 * Directly stops after the job has finished.
+		 */
+		DETACHED
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/JobClusterEntrypoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/JobClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/JobClusterEntrypoint.java
index e03c318..dc211d8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/JobClusterEntrypoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/JobClusterEntrypoint.java
@@ -18,52 +18,33 @@
 
 package org.apache.flink.runtime.entrypoint;
 
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.time.Time;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.WebOptions;
 import org.apache.flink.runtime.blob.BlobServer;
-import org.apache.flink.runtime.blob.TransientBlobCache;
 import org.apache.flink.runtime.blob.TransientBlobService;
-import org.apache.flink.runtime.clusterframework.ApplicationStatus;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph;
-import org.apache.flink.runtime.executiongraph.ErrorInfo;
+import org.apache.flink.runtime.concurrent.ScheduledExecutor;
+import org.apache.flink.runtime.dispatcher.ArchivedExecutionGraphStore;
+import org.apache.flink.runtime.dispatcher.Dispatcher;
+import org.apache.flink.runtime.dispatcher.DispatcherGateway;
+import org.apache.flink.runtime.dispatcher.MemoryArchivedExecutionGraphStore;
+import org.apache.flink.runtime.dispatcher.MiniDispatcher;
 import org.apache.flink.runtime.heartbeat.HeartbeatServices;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobmanager.OnCompletionActions;
-import org.apache.flink.runtime.jobmaster.JobManagerRunner;
-import org.apache.flink.runtime.jobmaster.JobManagerSharedServices;
-import org.apache.flink.runtime.jobmaster.JobMasterGateway;
-import org.apache.flink.runtime.jobmaster.JobMasterId;
-import org.apache.flink.runtime.jobmaster.JobMasterRestEndpoint;
+import org.apache.flink.runtime.jobmaster.MiniDispatcherRestEndpoint;
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.metrics.MetricRegistry;
-import org.apache.flink.runtime.resourcemanager.ResourceManager;
 import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
-import org.apache.flink.runtime.resourcemanager.ResourceManagerId;
 import org.apache.flink.runtime.rest.RestServerEndpointConfiguration;
 import org.apache.flink.runtime.rest.handler.RestHandlerConfiguration;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
-import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
 import org.apache.flink.runtime.webmonitor.retriever.LeaderGatewayRetriever;
 import org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever;
-import org.apache.flink.runtime.webmonitor.retriever.impl.AkkaQueryServiceRetriever;
-import org.apache.flink.runtime.webmonitor.retriever.impl.RpcGatewayRetriever;
-import org.apache.flink.util.ConfigurationException;
-import org.apache.flink.util.ExceptionUtils;
 import org.apache.flink.util.FlinkException;
-import org.apache.flink.util.Preconditions;
-
-import akka.actor.ActorSystem;
 
 import javax.annotation.Nullable;
 
-import java.net.InetSocketAddress;
+import java.io.IOException;
 import java.util.concurrent.Executor;
 
 /**
@@ -71,123 +52,24 @@ import java.util.concurrent.Executor;
  */
 public abstract class JobClusterEntrypoint extends ClusterEntrypoint {
 
-	private ResourceManager<?> resourceManager;
-
-	private JobManagerSharedServices jobManagerSharedServices;
-
-	private JobMasterRestEndpoint jobMasterRestEndpoint;
-
-	private LeaderRetrievalService jobMasterRetrievalService;
-
-	private LeaderRetrievalService resourceManagerRetrievalService;
-
-	private TransientBlobCache transientBlobCache;
-
-	private JobManagerRunner jobManagerRunner;
-
 	public JobClusterEntrypoint(Configuration configuration) {
 		super(configuration);
 	}
 
 	@Override
-	protected void startClusterComponents(
+	protected MiniDispatcherRestEndpoint createRestEndpoint(
 			Configuration configuration,
-			RpcService rpcService,
-			HighAvailabilityServices highAvailabilityServices,
-			BlobServer blobServer,
-			HeartbeatServices heartbeatServices,
-			MetricRegistry metricRegistry) throws Exception {
-
-		jobManagerSharedServices = JobManagerSharedServices.fromConfiguration(configuration, blobServer);
-
-		resourceManagerRetrievalService = highAvailabilityServices.getResourceManagerLeaderRetriever();
-
-		final LeaderGatewayRetriever<JobMasterGateway> jobMasterGatewayRetriever = new RpcGatewayRetriever<>(
-			rpcService,
-			JobMasterGateway.class,
-			JobMasterId::new,
-			10,
-			Time.milliseconds(50L));
-
-		final LeaderGatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever = new RpcGatewayRetriever<>(
-			rpcService,
-			ResourceManagerGateway.class,
-			ResourceManagerId::new,
-			10,
-			Time.milliseconds(50L));
-
-		// TODO: Remove once we have ported the MetricFetcher to the RpcEndpoint
-		final ActorSystem actorSystem = ((AkkaRpcService) rpcService).getActorSystem();
-		final Time timeout = Time.milliseconds(configuration.getLong(WebOptions.TIMEOUT));
-
-		final ClusterInformation clusterInformation = new ClusterInformation(rpcService.getAddress(), blobServer.getPort());
-
-		transientBlobCache = new TransientBlobCache(
-			configuration,
-			new InetSocketAddress(clusterInformation.getBlobServerHostname(), clusterInformation.getBlobServerPort()));
-
-		jobMasterRestEndpoint = createJobMasterRestEndpoint(
-			configuration,
-			jobMasterGatewayRetriever,
-			resourceManagerGatewayRetriever,
-			transientBlobCache,
-			rpcService.getExecutor(),
-			new AkkaQueryServiceRetriever(actorSystem, timeout),
-			highAvailabilityServices.getWebMonitorLeaderElectionService());
-
-		LOG.debug("Starting JobMaster REST endpoint.");
-		jobMasterRestEndpoint.start();
-
-		resourceManager = createResourceManager(
-			configuration,
-			ResourceID.generate(),
-			rpcService,
-			highAvailabilityServices,
-			heartbeatServices,
-			metricRegistry,
-			this,
-			clusterInformation,
-			jobMasterRestEndpoint.getRestAddress());
-
-		jobManagerRunner = createJobManagerRunner(
-			configuration,
-			ResourceID.generate(),
-			rpcService,
-			highAvailabilityServices,
-			jobManagerSharedServices,
-			heartbeatServices,
-			blobServer,
-			metricRegistry,
-			this,
-			jobMasterRestEndpoint.getRestAddress());
-
-		LOG.debug("Starting ResourceManager.");
-		resourceManager.start();
-		resourceManagerRetrievalService.start(resourceManagerGatewayRetriever);
-
-		LOG.debug("Starting JobManager.");
-		jobManagerRunner.start();
-
-		jobMasterRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(
-			jobManagerRunner.getJobGraph().getJobID(),
-			jobManagerRunner.getAddress());
-		jobMasterRetrievalService.start(jobMasterGatewayRetriever);
-	}
-
-	protected JobMasterRestEndpoint createJobMasterRestEndpoint(
-			Configuration configuration,
-			GatewayRetriever<JobMasterGateway> jobMasterGatewayRetriever,
-			GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
+			LeaderGatewayRetriever<DispatcherGateway> dispatcherGatewayRetriever,
+			LeaderGatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
 			TransientBlobService transientBlobService,
 			Executor executor,
 			MetricQueryServiceRetriever metricQueryServiceRetriever,
-			LeaderElectionService leaderElectionService) throws ConfigurationException {
-
+			LeaderElectionService leaderElectionService) throws Exception {
 		final RestHandlerConfiguration restHandlerConfiguration = RestHandlerConfiguration.fromConfiguration(configuration);
 
-		return new JobMasterRestEndpoint(
+		return new MiniDispatcherRestEndpoint(
 			RestServerEndpointConfiguration.fromConfiguration(configuration),
-			jobMasterGatewayRetriever,
+			dispatcherGatewayRetriever,
 			configuration,
 			restHandlerConfiguration,
 			resourceManagerGatewayRetriever,
@@ -196,155 +78,50 @@ public abstract class JobClusterEntrypoint extends ClusterEntrypoint {
 			metricQueryServiceRetriever,
 			leaderElectionService,
 			this);
+	}
 
+	@Override
+	protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
+			Configuration configuration,
+			ScheduledExecutor scheduledExecutor) throws IOException {
+		return new MemoryArchivedExecutionGraphStore();
 	}
 
-	protected JobManagerRunner createJobManagerRunner(
+	@Override
+	protected Dispatcher createDispatcher(
 			Configuration configuration,
-			ResourceID resourceId,
 			RpcService rpcService,
 			HighAvailabilityServices highAvailabilityServices,
-			JobManagerSharedServices jobManagerSharedServices,
-			HeartbeatServices heartbeatServices,
+			ResourceManagerGateway resourceManagerGateway,
 			BlobServer blobServer,
+			HeartbeatServices heartbeatServices,
 			MetricRegistry metricRegistry,
+			ArchivedExecutionGraphStore archivedExecutionGraphStore,
 			FatalErrorHandler fatalErrorHandler,
 			@Nullable String restAddress) throws Exception {
 
 		final JobGraph jobGraph = retrieveJobGraph(configuration);
 
-		return new JobManagerRunner(
-			resourceId,
-			jobGraph,
-			configuration,
+		final String executionModeValue = configuration.getString(EXECUTION_MODE);
+
+		final ExecutionMode executionMode = ExecutionMode.valueOf(executionModeValue);
+
+		return new MiniDispatcher(
 			rpcService,
+			Dispatcher.DISPATCHER_NAME,
+			configuration,
 			highAvailabilityServices,
-			heartbeatServices,
+			resourceManagerGateway,
 			blobServer,
-			jobManagerSharedServices,
+			heartbeatServices,
 			metricRegistry,
-			new TerminatingOnCompleteActions(jobGraph.getJobID()),
+			archivedExecutionGraphStore,
+			Dispatcher.DefaultJobManagerRunnerFactory.INSTANCE,
 			fatalErrorHandler,
-			restAddress);
-	}
-
-	@Override
-	protected void stopClusterComponents(boolean cleanupHaData) throws Exception {
-		Throwable exception = null;
-
-		if (jobMasterRestEndpoint != null) {
-			jobMasterRestEndpoint.shutdown(Time.seconds(10L));
-		}
-
-		if (jobMasterRetrievalService != null) {
-			try {
-				jobMasterRetrievalService.stop();
-			} catch (Throwable t) {
-				exception = ExceptionUtils.firstOrSuppressed(t, exception);
-			}
-		}
-
-		if (jobManagerRunner != null) {
-			try {
-				jobManagerRunner.shutdown();
-			} catch (Throwable t) {
-				exception = t;
-			}
-		}
-
-		if (jobManagerSharedServices != null) {
-			try {
-				jobManagerSharedServices.shutdown();
-			} catch (Throwable t) {
-				exception = ExceptionUtils.firstOrSuppressed(t, exception);
-			}
-		}
-
-		if (resourceManagerRetrievalService != null) {
-			try {
-				resourceManagerRetrievalService.stop();
-			} catch (Throwable t) {
-				exception = ExceptionUtils.firstOrSuppressed(t, exception);
-			}
-		}
-
-		if (resourceManager != null) {
-			try {
-				resourceManager.shutDown();
-			} catch (Throwable t) {
-				exception = ExceptionUtils.firstOrSuppressed(t, exception);
-			}
-		}
-
-		if (transientBlobCache != null) {
-			try {
-				transientBlobCache.close();
-			} catch (Throwable t) {
-				exception = ExceptionUtils.firstOrSuppressed(t, exception);
-			}
-		}
-
-		if (exception != null) {
-			throw new FlinkException("Could not properly shut down the job cluster entry point.", exception);
-		}
-	}
-
-	private void shutDownAndTerminate(
-			boolean cleanupHaData,
-			ApplicationStatus status,
-			@Nullable String optionalDiagnostics) {
-		try {
-			if (resourceManager != null) {
-				resourceManager.shutDownCluster(status, optionalDiagnostics);
-			}
-
-			shutDown(cleanupHaData);
-		} catch (Throwable t) {
-			LOG.error("Could not properly shut down cluster entrypoint.", t);
-		}
-
-		System.exit(SUCCESS_RETURN_CODE);
+			restAddress,
+			jobGraph,
+			executionMode);
 	}
 
-	protected abstract ResourceManager<?> createResourceManager(
-		Configuration configuration,
-		ResourceID resourceId,
-		RpcService rpcService,
-		HighAvailabilityServices highAvailabilityServices,
-		HeartbeatServices heartbeatServices,
-		MetricRegistry metricRegistry,
-		FatalErrorHandler fatalErrorHandler,
-		ClusterInformation clusterInformation,
-		@Nullable String webInterfaceUrl) throws Exception;
-
 	protected abstract JobGraph retrieveJobGraph(Configuration configuration) throws FlinkException;
-
-	private final class TerminatingOnCompleteActions implements OnCompletionActions {
-
-		private final JobID jobId;
-
-		private TerminatingOnCompleteActions(JobID jobId) {
-			this.jobId = Preconditions.checkNotNull(jobId);
-		}
-
-		@Override
-		public void jobReachedGloballyTerminalState(ArchivedExecutionGraph executionGraph) {
-			LOG.info("Job({}) finished.", jobId);
-
-			final ErrorInfo errorInfo = executionGraph.getFailureInfo();
-
-			if (errorInfo == null) {
-				shutDownAndTerminate(true, ApplicationStatus.SUCCEEDED, null);
-			} else {
-				shutDownAndTerminate(true, ApplicationStatus.FAILED, errorInfo.getExceptionAsString());
-			}
-		}
-
-		@Override
-		public void jobFinishedByOther() {
-			LOG.info("Job({}) was finished by another JobManager.", jobId);
-
-			shutDownAndTerminate(false, ApplicationStatus.UNKNOWN, "Job was finished by another master");
-		}
-	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java
index 0f41c9a..764356d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java
@@ -22,48 +22,33 @@ import org.apache.flink.api.common.time.Time;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.ConfigurationUtils;
 import org.apache.flink.configuration.JobManagerOptions;
-import org.apache.flink.configuration.WebOptions;
 import org.apache.flink.runtime.blob.BlobServer;
-import org.apache.flink.runtime.blob.TransientBlobCache;
 import org.apache.flink.runtime.blob.TransientBlobService;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.concurrent.ScheduledExecutor;
 import org.apache.flink.runtime.dispatcher.ArchivedExecutionGraphStore;
 import org.apache.flink.runtime.dispatcher.Dispatcher;
 import org.apache.flink.runtime.dispatcher.DispatcherGateway;
-import org.apache.flink.runtime.dispatcher.DispatcherId;
 import org.apache.flink.runtime.dispatcher.DispatcherRestEndpoint;
 import org.apache.flink.runtime.dispatcher.FileArchivedExecutionGraphStore;
 import org.apache.flink.runtime.dispatcher.StandaloneDispatcher;
 import org.apache.flink.runtime.heartbeat.HeartbeatServices;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.metrics.MetricRegistry;
-import org.apache.flink.runtime.resourcemanager.ResourceManager;
 import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
-import org.apache.flink.runtime.resourcemanager.ResourceManagerId;
 import org.apache.flink.runtime.rest.RestServerEndpointConfiguration;
 import org.apache.flink.runtime.rest.handler.RestHandlerConfiguration;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
 import org.apache.flink.runtime.webmonitor.retriever.LeaderGatewayRetriever;
 import org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever;
-import org.apache.flink.runtime.webmonitor.retriever.impl.AkkaQueryServiceRetriever;
-import org.apache.flink.runtime.webmonitor.retriever.impl.RpcGatewayRetriever;
-import org.apache.flink.util.ExceptionUtils;
-import org.apache.flink.util.FlinkException;
 
 import org.apache.flink.shaded.guava18.com.google.common.base.Ticker;
 
-import akka.actor.ActorSystem;
-
 import javax.annotation.Nullable;
 
 import java.io.File;
 import java.io.IOException;
-import java.net.InetSocketAddress;
 import java.util.concurrent.Executor;
 
 /**
@@ -71,110 +56,12 @@ import java.util.concurrent.Executor;
  */
 public abstract class SessionClusterEntrypoint extends ClusterEntrypoint {
 
-	private ResourceManager<?> resourceManager;
-
-	private Dispatcher dispatcher;
-
-	private LeaderRetrievalService dispatcherLeaderRetrievalService;
-
-	private LeaderRetrievalService resourceManagerRetrievalService;
-
-	private DispatcherRestEndpoint dispatcherRestEndpoint;
-
-	private ArchivedExecutionGraphStore archivedExecutionGraphStore;
-
-	private TransientBlobCache transientBlobCache;
-
 	public SessionClusterEntrypoint(Configuration configuration) {
 		super(configuration);
 	}
 
 	@Override
-	protected void startClusterComponents(
-			Configuration configuration,
-			RpcService rpcService,
-			HighAvailabilityServices highAvailabilityServices,
-			BlobServer blobServer,
-			HeartbeatServices heartbeatServices,
-			MetricRegistry metricRegistry) throws Exception {
-
-		archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, rpcService.getScheduledExecutor());
-
-		dispatcherLeaderRetrievalService = highAvailabilityServices.getDispatcherLeaderRetriever();
-
-		resourceManagerRetrievalService = highAvailabilityServices.getResourceManagerLeaderRetriever();
-
-		LeaderGatewayRetriever<DispatcherGateway> dispatcherGatewayRetriever = new RpcGatewayRetriever<>(
-			rpcService,
-			DispatcherGateway.class,
-			DispatcherId::new,
-			10,
-			Time.milliseconds(50L));
-
-		LeaderGatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever = new RpcGatewayRetriever<>(
-			rpcService,
-			ResourceManagerGateway.class,
-			ResourceManagerId::new,
-			10,
-			Time.milliseconds(50L));
-
-		// TODO: Remove once we have ported the MetricFetcher to the RpcEndpoint
-		final ActorSystem actorSystem = ((AkkaRpcService) rpcService).getActorSystem();
-		final Time timeout = Time.milliseconds(configuration.getLong(WebOptions.TIMEOUT));
-
-		final ClusterInformation clusterInformation = new ClusterInformation(
-			rpcService.getAddress(),
-			blobServer.getPort());
-
-		transientBlobCache = new TransientBlobCache(
-			configuration,
-			new InetSocketAddress(clusterInformation.getBlobServerHostname(), clusterInformation.getBlobServerPort()));
-
-		dispatcherRestEndpoint = createDispatcherRestEndpoint(
-			configuration,
-			dispatcherGatewayRetriever,
-			resourceManagerGatewayRetriever,
-			transientBlobCache,
-			rpcService.getExecutor(),
-			new AkkaQueryServiceRetriever(actorSystem, timeout),
-			highAvailabilityServices.getWebMonitorLeaderElectionService());
-
-		LOG.debug("Starting Dispatcher REST endpoint.");
-		dispatcherRestEndpoint.start();
-
-		resourceManager = createResourceManager(
-			configuration,
-			ResourceID.generate(),
-			rpcService,
-			highAvailabilityServices,
-			heartbeatServices,
-			metricRegistry,
-			this,
-			clusterInformation,
-			dispatcherRestEndpoint.getRestAddress());
-
-		dispatcher = createDispatcher(
-			configuration,
-			rpcService,
-			highAvailabilityServices,
-			resourceManager.getSelfGateway(ResourceManagerGateway.class),
-			blobServer,
-			heartbeatServices,
-			metricRegistry,
-			archivedExecutionGraphStore,
-			this,
-			dispatcherRestEndpoint.getRestAddress());
-
-		LOG.debug("Starting ResourceManager.");
-		resourceManager.start();
-		resourceManagerRetrievalService.start(resourceManagerGatewayRetriever);
-
-		LOG.debug("Starting Dispatcher.");
-		dispatcher.start();
-		dispatcherLeaderRetrievalService.start(dispatcherGatewayRetriever);
-	}
-
-	private ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
+	protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
 			Configuration configuration,
 			ScheduledExecutor scheduledExecutor) throws IOException {
 		final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);
@@ -191,74 +78,14 @@ public abstract class SessionClusterEntrypoint extends ClusterEntrypoint {
 	}
 
 	@Override
-	protected void stopClusterComponents(boolean cleanupHaData) throws Exception {
-		Throwable exception = null;
-
-		if (dispatcherRestEndpoint != null) {
-			dispatcherRestEndpoint.shutdown(Time.seconds(10L));
-		}
-
-		if (dispatcherLeaderRetrievalService != null) {
-			try {
-				dispatcherLeaderRetrievalService.stop();
-			} catch (Throwable t) {
-				exception = ExceptionUtils.firstOrSuppressed(t, exception);
-			}
-		}
-
-		if (dispatcher != null) {
-			try {
-				dispatcher.shutDown();
-			} catch (Throwable t) {
-				exception = ExceptionUtils.firstOrSuppressed(t, exception);
-			}
-		}
-
-		if (resourceManagerRetrievalService != null) {
-			try {
-				resourceManagerRetrievalService.stop();
-			} catch (Throwable t) {
-				exception = ExceptionUtils.firstOrSuppressed(t, exception);
-			}
-		}
-
-		if (resourceManager != null) {
-			try {
-				resourceManager.shutDown();
-			} catch (Throwable t) {
-				exception = ExceptionUtils.firstOrSuppressed(t, exception);
-			}
-		}
-
-		if (archivedExecutionGraphStore != null) {
-			try {
-				archivedExecutionGraphStore.close();
-			} catch (Throwable t) {
-				exception = ExceptionUtils.firstOrSuppressed(t, exception);
-			}
-		}
-
-		if (transientBlobCache != null) {
-			try {
-				transientBlobCache.close();
-			} catch (Throwable t) {
-				exception = ExceptionUtils.firstOrSuppressed(t, exception);
-			}
-		}
-
-		if (exception != null) {
-			throw new FlinkException("Could not properly shut down the session cluster entry point.", exception);
-		}
-	}
-
-	protected DispatcherRestEndpoint createDispatcherRestEndpoint(
-		Configuration configuration,
-		LeaderGatewayRetriever<DispatcherGateway> dispatcherGatewayRetriever,
-		LeaderGatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
+	protected DispatcherRestEndpoint createRestEndpoint(
+			Configuration configuration,
+			LeaderGatewayRetriever<DispatcherGateway> dispatcherGatewayRetriever,
+			LeaderGatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
 			TransientBlobService transientBlobService,
-		Executor executor,
-		MetricQueryServiceRetriever metricQueryServiceRetriever,
-		LeaderElectionService leaderElectionService) throws Exception {
+			Executor executor,
+			MetricQueryServiceRetriever metricQueryServiceRetriever,
+			LeaderElectionService leaderElectionService) throws Exception {
 
 		final RestHandlerConfiguration restHandlerConfiguration = RestHandlerConfiguration.fromConfiguration(configuration);
 
@@ -275,6 +102,7 @@ public abstract class SessionClusterEntrypoint extends ClusterEntrypoint {
 			this);
 	}
 
+	@Override
 	protected Dispatcher createDispatcher(
 		Configuration configuration,
 		RpcService rpcService,
@@ -298,18 +126,8 @@ public abstract class SessionClusterEntrypoint extends ClusterEntrypoint {
 			heartbeatServices,
 			metricRegistry,
 			archivedExecutionGraphStore,
+			Dispatcher.DefaultJobManagerRunnerFactory.INSTANCE,
 			fatalErrorHandler,
 			restAddress);
 	}
-
-	protected abstract ResourceManager<?> createResourceManager(
-		Configuration configuration,
-		ResourceID resourceId,
-		RpcService rpcService,
-		HighAvailabilityServices highAvailabilityServices,
-		HeartbeatServices heartbeatServices,
-		MetricRegistry metricRegistry,
-		FatalErrorHandler fatalErrorHandler,
-		ClusterInformation clusterInformation,
-		@Nullable String webInterfaceUrl) throws Exception;
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterRestEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterRestEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterRestEndpoint.java
deleted file mode 100644
index b0fa46d..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterRestEndpoint.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.jobmaster;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.blob.TransientBlobService;
-import org.apache.flink.runtime.leaderelection.LeaderElectionService;
-import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
-import org.apache.flink.runtime.rest.RestServerEndpointConfiguration;
-import org.apache.flink.runtime.rest.handler.RestHandlerConfiguration;
-import org.apache.flink.runtime.rpc.FatalErrorHandler;
-import org.apache.flink.runtime.webmonitor.WebMonitorEndpoint;
-import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
-import org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever;
-
-import java.util.concurrent.Executor;
-
-/**
- * REST endpoint for the {@link JobMaster}.
- */
-public class JobMasterRestEndpoint extends WebMonitorEndpoint<JobMasterGateway> {
-
-	public JobMasterRestEndpoint(
-			RestServerEndpointConfiguration endpointConfiguration,
-			GatewayRetriever<JobMasterGateway> leaderRetriever,
-			Configuration clusterConfiguration,
-			RestHandlerConfiguration restConfiguration,
-			GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
-			TransientBlobService transientBlobService,
-			Executor executor,
-			MetricQueryServiceRetriever metricQueryServiceRetriever,
-			LeaderElectionService leaderElectionService,
-			FatalErrorHandler fatalErrorHandler) {
-		super(
-			endpointConfiguration,
-			leaderRetriever,
-			clusterConfiguration,
-			restConfiguration,
-			resourceManagerRetriever,
-			transientBlobService,
-			executor,
-			metricQueryServiceRetriever,
-			leaderElectionService,
-			fatalErrorHandler);
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniDispatcherRestEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniDispatcherRestEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniDispatcherRestEndpoint.java
new file mode 100644
index 0000000..54ad526
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniDispatcherRestEndpoint.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.blob.TransientBlobService;
+import org.apache.flink.runtime.entrypoint.JobClusterEntrypoint;
+import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
+import org.apache.flink.runtime.rest.RestServerEndpointConfiguration;
+import org.apache.flink.runtime.rest.handler.RestHandlerConfiguration;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
+import org.apache.flink.runtime.webmonitor.RestfulGateway;
+import org.apache.flink.runtime.webmonitor.WebMonitorEndpoint;
+import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
+import org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever;
+
+import java.util.concurrent.Executor;
+
+/**
+ * REST endpoint for the {@link JobClusterEntrypoint}.
+ */
+public class MiniDispatcherRestEndpoint extends WebMonitorEndpoint<RestfulGateway> {
+
+	public MiniDispatcherRestEndpoint(
+			RestServerEndpointConfiguration endpointConfiguration,
+			GatewayRetriever<? extends RestfulGateway> leaderRetriever,
+			Configuration clusterConfiguration,
+			RestHandlerConfiguration restConfiguration,
+			GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
+			TransientBlobService transientBlobService,
+			Executor executor,
+			MetricQueryServiceRetriever metricQueryServiceRetriever,
+			LeaderElectionService leaderElectionService,
+			FatalErrorHandler fatalErrorHandler) {
+		super(
+			endpointConfiguration,
+			leaderRetriever,
+			clusterConfiguration,
+			restConfiguration,
+			resourceManagerRetriever,
+			transientBlobService,
+			executor,
+			metricQueryServiceRetriever,
+			leaderElectionService,
+			fatalErrorHandler);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/BlobServerPortHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/BlobServerPortHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/BlobServerPortHandler.java
index 1d14563..4b5fa89 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/BlobServerPortHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/BlobServerPortHandler.java
@@ -45,7 +45,7 @@ public final class BlobServerPortHandler extends AbstractRestHandler<DispatcherG
 
 	public BlobServerPortHandler(
 			CompletableFuture<String> localRestAddress,
-			GatewayRetriever<DispatcherGateway> leaderRetriever,
+			GatewayRetriever<? extends DispatcherGateway> leaderRetriever,
 			Time timeout,
 			Map<String, String> headers) {
 		super(localRestAddress, leaderRetriever, timeout, headers, BlobServerPortHeaders.getInstance());

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobSubmitHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobSubmitHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobSubmitHandler.java
index efb162e..ac0a17b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobSubmitHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobSubmitHandler.java
@@ -46,7 +46,7 @@ public final class JobSubmitHandler extends AbstractRestHandler<DispatcherGatewa
 
 	public JobSubmitHandler(
 			CompletableFuture<String> localRestAddress,
-			GatewayRetriever<DispatcherGateway> leaderRetriever,
+			GatewayRetriever<? extends DispatcherGateway> leaderRetriever,
 			Time timeout,
 			Map<String, String> headers) {
 		super(localRestAddress, leaderRetriever, timeout, headers, JobSubmitHeaders.getInstance());

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/legacy/files/StaticFileServerHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/legacy/files/StaticFileServerHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/legacy/files/StaticFileServerHandler.java
index 650647b..5159e12 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/legacy/files/StaticFileServerHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/legacy/files/StaticFileServerHandler.java
@@ -110,7 +110,7 @@ public class StaticFileServerHandler<T extends RestfulGateway> extends RedirectH
 	private final File rootPath;
 
 	public StaticFileServerHandler(
-			GatewayRetriever<T> retriever,
+			GatewayRetriever<? extends T> retriever,
 			CompletableFuture<String> localJobManagerAddressFuture,
 			Time timeout,
 			File rootPath) throws IOException {

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorRegistrationSuccess.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorRegistrationSuccess.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorRegistrationSuccess.java
index dc8d599..1b1c1e5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorRegistrationSuccess.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorRegistrationSuccess.java
@@ -44,7 +44,7 @@ public final class TaskExecutorRegistrationSuccess extends RegistrationResponse.
 
 	/**
 	 * Create a new {@code TaskExecutorRegistrationSuccess} message.
-	 * 
+	 *
 	 * @param registrationId The ID that the ResourceManager assigned the registration.
 	 * @param resourceManagerResourceId The unique ID that identifies the ResourceManager.
 	 * @param heartbeatInterval The interval in which the ResourceManager will heartbeat the TaskExecutor.


[16/22] flink git commit: [FLINK-7713][flip6] Implement JarUploadHandler

Posted by tr...@apache.org.
[FLINK-7713][flip6] Implement JarUploadHandler

This closes #5442.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/39df56d6
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/39df56d6
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/39df56d6

Branch: refs/heads/master
Commit: 39df56d6aebb3911f19593dbf120d42789e8db5a
Parents: e3e601b
Author: gyao <ga...@data-artisans.com>
Authored: Wed Feb 14 16:48:47 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Thu Feb 15 08:44:20 2018 +0100

----------------------------------------------------------------------
 .../runtime/webmonitor/WebRuntimeMonitor.java   |  12 +-
 .../handlers/JarAccessDeniedHandler.java        |  55 --
 .../webmonitor/handlers/JarActionHandler.java   | 258 ---------
 .../webmonitor/handlers/JarDeleteHandler.java   | 102 ----
 .../webmonitor/handlers/JarListHandler.java     | 164 ------
 .../webmonitor/handlers/JarPlanHandler.java     |  73 ---
 .../webmonitor/handlers/JarRunHandler.java      |  95 ----
 .../webmonitor/handlers/JarUploadHandler.java   | 115 ++--
 .../handlers/JarUploadMessageHeaders.java       |  63 +++
 .../handlers/JarUploadResponseBody.java         |  60 ++
 .../handlers/legacy/JarAccessDeniedHandler.java |  55 ++
 .../handlers/legacy/JarActionHandler.java       | 258 +++++++++
 .../handlers/legacy/JarDeleteHandler.java       | 102 ++++
 .../handlers/legacy/JarListHandler.java         | 164 ++++++
 .../handlers/legacy/JarPlanHandler.java         |  73 +++
 .../handlers/legacy/JarRunHandler.java          |  95 ++++
 .../handlers/legacy/JarUploadHandler.java       |  84 +++
 .../runtime/webmonitor/WebMonitorUtilsTest.java |  25 +
 .../handlers/JarAccessDeniedHandlerTest.java    |  46 --
 .../handlers/JarActionHandlerTest.java          |  94 ----
 .../handlers/JarDeleteHandlerTest.java          |  37 --
 .../webmonitor/handlers/JarListHandlerTest.java |  37 --
 .../webmonitor/handlers/JarPlanHandlerTest.java |  37 --
 .../webmonitor/handlers/JarRunHandlerTest.java  |  39 --
 .../handlers/JarUploadHandlerTest.java          | 113 +++-
 .../handlers/JarUploadResponseBodyTest.java     |  46 ++
 .../legacy/JarAccessDeniedHandlerTest.java      |  46 ++
 .../handlers/legacy/JarActionHandlerTest.java   |  94 ++++
 .../handlers/legacy/JarDeleteHandlerTest.java   |  37 ++
 .../handlers/legacy/JarListHandlerTest.java     |  37 ++
 .../handlers/legacy/JarPlanHandlerTest.java     |  37 ++
 .../handlers/legacy/JarRunHandlerTest.java      |  39 ++
 .../handlers/legacy/JarUploadHandlerTest.java   |  37 ++
 .../dispatcher/DispatcherRestEndpoint.java      |  18 +
 .../flink/runtime/rest/AbstractHandler.java     |  21 +-
 .../flink/runtime/rest/FileUploadHandler.java   | 159 ++++++
 .../flink/runtime/rest/RestServerEndpoint.java  |  15 +-
 .../rest/RestServerEndpointConfiguration.java   |  34 +-
 .../flink/runtime/rest/messages/FileUpload.java |  44 ++
 .../runtime/webmonitor/WebMonitorEndpoint.java  |   4 +-
 .../runtime/webmonitor/WebMonitorUtils.java     |  85 ++-
 .../runtime/rest/FileUploadHandlerTest.java     |  67 +++
 .../flink/runtime/rest/RestEndpointITCase.java  | 383 -------------
 .../runtime/rest/RestServerEndpointITCase.java  | 541 +++++++++++++++++++
 44 files changed, 2500 insertions(+), 1500 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
index c0c6ac1..6abe0e7 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
@@ -72,12 +72,12 @@ import org.apache.flink.runtime.rest.handler.legacy.metrics.JobVertexMetricsHand
 import org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher;
 import org.apache.flink.runtime.rest.handler.legacy.metrics.SubtaskMetricsHandler;
 import org.apache.flink.runtime.rest.handler.legacy.metrics.TaskManagerMetricsHandler;
-import org.apache.flink.runtime.webmonitor.handlers.JarAccessDeniedHandler;
-import org.apache.flink.runtime.webmonitor.handlers.JarDeleteHandler;
-import org.apache.flink.runtime.webmonitor.handlers.JarListHandler;
-import org.apache.flink.runtime.webmonitor.handlers.JarPlanHandler;
-import org.apache.flink.runtime.webmonitor.handlers.JarRunHandler;
-import org.apache.flink.runtime.webmonitor.handlers.JarUploadHandler;
+import org.apache.flink.runtime.webmonitor.handlers.legacy.JarAccessDeniedHandler;
+import org.apache.flink.runtime.webmonitor.handlers.legacy.JarDeleteHandler;
+import org.apache.flink.runtime.webmonitor.handlers.legacy.JarListHandler;
+import org.apache.flink.runtime.webmonitor.handlers.legacy.JarPlanHandler;
+import org.apache.flink.runtime.webmonitor.handlers.legacy.JarRunHandler;
+import org.apache.flink.runtime.webmonitor.handlers.legacy.JarUploadHandler;
 import org.apache.flink.runtime.webmonitor.history.JsonArchivist;
 import org.apache.flink.runtime.webmonitor.retriever.LeaderGatewayRetriever;
 import org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever;

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarAccessDeniedHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarAccessDeniedHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarAccessDeniedHandler.java
deleted file mode 100644
index c95cc32..0000000
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarAccessDeniedHandler.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.webmonitor.handlers;
-
-import org.apache.flink.runtime.jobmaster.JobManagerGateway;
-import org.apache.flink.runtime.rest.handler.legacy.AbstractJsonRequestHandler;
-
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Executor;
-
-/**
- * Handler to deny access to jar-related REST calls.
- */
-public class JarAccessDeniedHandler extends AbstractJsonRequestHandler {
-
-	private static final String ERROR_MESSAGE = "{\"error\": \"Web submission interface is not " +
-			"available for this cluster. To enable it, set the configuration key ' jobmanager.web.submit.enable.'\"}";
-
-	public JarAccessDeniedHandler(Executor executor) {
-		super(executor);
-	}
-
-	@Override
-	public String[] getPaths() {
-		return new String[]{
-			JarListHandler.JAR_LIST_REST_PATH,
-			JarPlanHandler.JAR_PLAN_REST_PATH,
-			JarRunHandler.JAR_RUN_REST_PATH,
-			JarUploadHandler.JAR_UPLOAD_REST_PATH,
-			JarDeleteHandler.JAR_DELETE_REST_PATH
-		};
-	}
-
-	@Override
-	public CompletableFuture<String> handleJsonRequest(Map<String, String> pathParams, Map<String, String> queryParams, JobManagerGateway jobManagerGateway) {
-		return CompletableFuture.completedFuture(ERROR_MESSAGE);
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarActionHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarActionHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarActionHandler.java
deleted file mode 100644
index eea3a87..0000000
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarActionHandler.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.webmonitor.handlers;
-
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.client.program.ClusterClient;
-import org.apache.flink.client.program.PackagedProgram;
-import org.apache.flink.client.program.ProgramInvocationException;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.Optimizer;
-import org.apache.flink.optimizer.costs.DefaultCostEstimator;
-import org.apache.flink.optimizer.plan.FlinkPlan;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.StreamingPlan;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
-import org.apache.flink.runtime.rest.handler.legacy.AbstractJsonRequestHandler;
-import org.apache.flink.runtime.rest.handler.legacy.JsonFactory;
-import org.apache.flink.runtime.webmonitor.WebRuntimeMonitor;
-import org.apache.flink.util.ExceptionUtils;
-
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.StringWriter;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Executor;
-
-/**
- * Abstract handler for fetching plan for a jar or running a jar.
- */
-public abstract class JarActionHandler extends AbstractJsonRequestHandler {
-
-	private final File jarDir;
-
-	public JarActionHandler(Executor executor, File jarDirectory) {
-		super(executor);
-		jarDir = jarDirectory;
-	}
-
-	protected Tuple2<JobGraph, ClassLoader> getJobGraphAndClassLoader(JarActionHandlerConfig config) throws Exception {
-		// generate the graph
-		JobGraph graph = null;
-
-		if (!jarDir.exists()) {
-			WebRuntimeMonitor.logExternalUploadDirDeletion(jarDir);
-			try {
-				WebRuntimeMonitor.checkAndCreateUploadDir(jarDir);
-			} catch (IOException ioe) {
-				// the following code will throw an exception since the jar can't be found
-			}
-		}
-
-		PackagedProgram program = new PackagedProgram(
-				new File(jarDir, config.getJarFile()),
-				config.getEntryClass(),
-				config.getProgramArgs());
-		ClassLoader classLoader = program.getUserCodeClassLoader();
-
-		Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration());
-		FlinkPlan plan = ClusterClient.getOptimizedPlan(optimizer, program, config.getParallelism());
-
-		if (plan instanceof StreamingPlan) {
-			graph = ((StreamingPlan) plan).getJobGraph();
-		} else if (plan instanceof OptimizedPlan) {
-			graph = new JobGraphGenerator().compileJobGraph((OptimizedPlan) plan);
-		}
-		if (graph == null) {
-			throw new CompilerException("A valid job graph couldn't be generated for the jar.");
-		}
-
-		// Set the savepoint settings
-		graph.setSavepointRestoreSettings(config.getSavepointRestoreSettings());
-
-		for (URL jar : program.getAllLibraries()) {
-			try {
-				graph.addJar(new Path(jar.toURI()));
-			}
-			catch (URISyntaxException e) {
-				throw new ProgramInvocationException("Invalid jar path. Unexpected error. :(");
-			}
-		}
-		return Tuple2.of(graph, classLoader);
-	}
-
-	protected String sendError(Exception e) throws Exception {
-		StringWriter writer = new StringWriter();
-		JsonGenerator gen = JsonFactory.JACKSON_FACTORY.createGenerator(writer);
-
-		gen.writeStartObject();
-		gen.writeStringField("error", ExceptionUtils.stringifyException(e));
-		gen.writeEndObject();
-		gen.close();
-
-		return writer.toString();
-	}
-
-	/**
-	 * Wrapper for all configuration that is parsed from query and path args.
-	 */
-	@VisibleForTesting
-	static class JarActionHandlerConfig {
-
-		private final String jarFile;
-		private final String[] programArgs;
-		private final String entryClass;
-		private final int parallelism;
-		private final SavepointRestoreSettings savepointRestoreSettings;
-
-		JarActionHandlerConfig(
-				String jarFile,
-				String[] programArgs,
-				String entryClass,
-				int parallelism,
-				SavepointRestoreSettings savepointRestoreSettings) {
-
-			this.jarFile = jarFile;
-			this.programArgs = programArgs;
-			this.entryClass = entryClass;
-			this.parallelism = parallelism;
-			this.savepointRestoreSettings = savepointRestoreSettings;
-		}
-
-		public String getJarFile() {
-			return jarFile;
-		}
-
-		public String[] getProgramArgs() {
-			return programArgs;
-		}
-
-		public String getEntryClass() {
-			return entryClass;
-		}
-
-		public int getParallelism() {
-			return parallelism;
-		}
-
-		public SavepointRestoreSettings getSavepointRestoreSettings() {
-			return savepointRestoreSettings;
-		}
-
-		public static JarActionHandlerConfig fromParams(Map<String, String> pathParams, Map<String, String> queryParams) {
-			// Jar file
-			String jarFile = pathParams.get("jarid");
-			if (jarFile == null) {
-				throw new IllegalArgumentException("No jarid was provided.");
-			}
-
-			// Program args
-			String[] programArgs = new String[0];
-			String programArgsOpt = queryParams.get("program-args");
-			if (programArgsOpt != null && !programArgsOpt.equals("")) {
-				List<String> args = tokenizeArguments(programArgsOpt);
-				programArgs = args.toArray(new String[args.size()]);
-			}
-
-			// Entry class
-			String entryClass = null;
-			String entryClassOpt = queryParams.get("entry-class");
-			if (entryClassOpt != null && !entryClassOpt.equals("")) {
-				entryClass = entryClassOpt;
-			}
-
-			// Parallelism
-			int parallelism = 1;
-			String parallelismOpt = queryParams.get("parallelism");
-			if (parallelismOpt != null && !parallelismOpt.equals("")) {
-				parallelism = Integer.parseInt(parallelismOpt);
-				if (parallelism < 1) {
-					throw new IllegalArgumentException("Parallelism must be a positive number.");
-				}
-			}
-
-			// Savepoint restore settings
-			SavepointRestoreSettings savepointSettings = SavepointRestoreSettings.none();
-			String savepointPath = queryParams.get("savepointPath");
-			if (savepointPath != null && !savepointPath.equals("")) {
-				String allowNonRestoredOpt = queryParams.get("allowNonRestoredState");
-				boolean allowNonRestoredState = allowNonRestoredOpt != null && allowNonRestoredOpt.equals("true");
-				savepointSettings = SavepointRestoreSettings.forPath(savepointPath, allowNonRestoredState);
-			}
-
-			return new JarActionHandlerConfig(jarFile, programArgs, entryClass, parallelism, savepointSettings);
-		}
-
-		/**
-		 * Utility method that takes the given arguments, splits them at the whitespaces (space and tab) and
-		 * turns them into an array of Strings. Other than the <tt>StringTokenizer</tt>, this method
-		 * takes care of quotes, such that quoted passages end up being one string.
-		 *
-		 * @param args
-		 *        The string to be split.
-		 * @return The array of split strings.
-		 */
-		private static List<String> tokenizeArguments(String args) {
-			List<String> list = new ArrayList<String>();
-			StringBuilder curr = new StringBuilder();
-
-			int pos = 0;
-			boolean quoted = false;
-
-			while (pos < args.length()) {
-				char c = args.charAt(pos);
-				if ((c == ' ' || c == '\t') && !quoted) {
-					if (curr.length() > 0) {
-						list.add(curr.toString());
-						curr.setLength(0);
-					}
-				} else if (c == '"') {
-					quoted = !quoted;
-				} else {
-					curr.append(c);
-				}
-
-				pos++;
-			}
-
-			if (quoted) {
-				throw new IllegalArgumentException("Unterminated quoted string.");
-			}
-
-			if (curr.length() > 0) {
-				list.add(curr.toString());
-			}
-
-			return list;
-		}
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandler.java
deleted file mode 100644
index c1a5f60..0000000
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandler.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.webmonitor.handlers;
-
-import org.apache.flink.runtime.jobmaster.JobManagerGateway;
-import org.apache.flink.runtime.rest.handler.legacy.AbstractJsonRequestHandler;
-import org.apache.flink.runtime.rest.handler.legacy.JsonFactory;
-import org.apache.flink.runtime.webmonitor.WebRuntimeMonitor;
-import org.apache.flink.util.FlinkException;
-
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.StringWriter;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import java.util.concurrent.Executor;
-
-/**
- * Handles requests for deletion of jars.
- */
-public class JarDeleteHandler extends AbstractJsonRequestHandler {
-
-	static final String JAR_DELETE_REST_PATH = "/jars/:jarid";
-
-	private final File jarDir;
-
-	public JarDeleteHandler(Executor executor, File jarDirectory) {
-		super(executor);
-		jarDir = jarDirectory;
-	}
-
-	@Override
-	public String[] getPaths() {
-		return new String[]{JAR_DELETE_REST_PATH};
-	}
-
-	@Override
-	public CompletableFuture<String> handleJsonRequest(Map<String, String> pathParams, Map<String, String> queryParams, JobManagerGateway jobManagerGateway) {
-		final String file = pathParams.get("jarid");
-		return CompletableFuture.supplyAsync(
-			() -> {
-				try {
-					File[] list = jarDir.listFiles(new FilenameFilter() {
-						@Override
-						public boolean accept(File dir, String name) {
-							return name.equals(file);
-						}
-					});
-
-					if (list == null) {
-						WebRuntimeMonitor.logExternalUploadDirDeletion(jarDir);
-						try {
-							WebRuntimeMonitor.checkAndCreateUploadDir(jarDir);
-						} catch (IOException ioe) {
-							// entire directory doesn't exist anymore, continue as if deletion succeeded
-						}
-						list = new File[0];
-					}
-
-					boolean success = false;
-					for (File f: list) {
-						// although next to impossible for multiple files, we still delete them.
-						success = success || f.delete();
-					}
-					StringWriter writer = new StringWriter();
-					JsonGenerator gen = JsonFactory.JACKSON_FACTORY.createGenerator(writer);
-					gen.writeStartObject();
-					if (!success) {
-						// this seems to always fail on Windows.
-						gen.writeStringField("error", "The requested jar couldn't be deleted. Please try again.");
-					}
-					gen.writeEndObject();
-					gen.close();
-					return writer.toString();
-				}
-				catch (Exception e) {
-					throw new CompletionException(new FlinkException("Failed to delete jar id " + pathParams.get("jarid") + '.', e));
-				}
-			},
-			executor);
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandler.java
deleted file mode 100644
index 66cc7f5..0000000
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandler.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.webmonitor.handlers;
-
-import org.apache.flink.client.program.PackagedProgram;
-import org.apache.flink.runtime.jobmaster.JobManagerGateway;
-import org.apache.flink.runtime.rest.handler.legacy.AbstractJsonRequestHandler;
-import org.apache.flink.runtime.rest.handler.legacy.JsonFactory;
-import org.apache.flink.runtime.webmonitor.RuntimeMonitorHandler;
-import org.apache.flink.runtime.webmonitor.WebRuntimeMonitor;
-import org.apache.flink.util.FlinkException;
-
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.StringWriter;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import java.util.concurrent.Executor;
-import java.util.jar.JarFile;
-import java.util.jar.Manifest;
-
-/**
- * Handle request for listing uploaded jars.
- */
-public class JarListHandler extends AbstractJsonRequestHandler {
-
-	static final String JAR_LIST_REST_PATH = "/jars";
-
-	private final File jarDir;
-
-	public  JarListHandler(Executor executor, File jarDirectory) {
-		super(executor);
-		jarDir = jarDirectory;
-	}
-
-	@Override
-	public String[] getPaths() {
-		return new String[]{JAR_LIST_REST_PATH};
-	}
-
-	@Override
-	public CompletableFuture<String> handleJsonRequest(Map<String, String> pathParams, Map<String, String> queryParams, JobManagerGateway jobManagerGateway) {
-		return CompletableFuture.supplyAsync(
-			() -> {
-				try {
-					StringWriter writer = new StringWriter();
-					JsonGenerator gen = JsonFactory.JACKSON_FACTORY.createGenerator(writer);
-
-					gen.writeStartObject();
-					gen.writeStringField("address", queryParams.get(RuntimeMonitorHandler.WEB_MONITOR_ADDRESS_KEY));
-					gen.writeArrayFieldStart("files");
-
-					File[] list = jarDir.listFiles(new FilenameFilter() {
-						@Override
-						public boolean accept(File dir, String name) {
-							return name.endsWith(".jar");
-						}
-					});
-
-					if (list == null) {
-						WebRuntimeMonitor.logExternalUploadDirDeletion(jarDir);
-						try {
-							WebRuntimeMonitor.checkAndCreateUploadDir(jarDir);
-						} catch (IOException ioe) {
-							// re-throwing an exception here breaks the UI
-						}
-						list = new File[0];
-					}
-
-					// last modified ascending order
-					Arrays.sort(list, (f1, f2) -> Long.compare(f2.lastModified(), f1.lastModified()));
-
-					for (File f : list) {
-						// separate the uuid and the name parts.
-						String id = f.getName();
-
-						int startIndex = id.indexOf("_");
-						if (startIndex < 0) {
-							continue;
-						}
-						String name = id.substring(startIndex + 1);
-						if (name.length() < 5 || !name.endsWith(".jar")) {
-							continue;
-						}
-
-						gen.writeStartObject();
-						gen.writeStringField("id", id);
-						gen.writeStringField("name", name);
-						gen.writeNumberField("uploaded", f.lastModified());
-						gen.writeArrayFieldStart("entry");
-
-						String[] classes = new String[0];
-						try (JarFile jar = new JarFile(f)) {
-							Manifest manifest = jar.getManifest();
-							String assemblerClass = null;
-
-							if (manifest != null) {
-								assemblerClass = manifest.getMainAttributes().getValue(PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS);
-								if (assemblerClass == null) {
-									assemblerClass = manifest.getMainAttributes().getValue(PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS);
-								}
-							}
-							if (assemblerClass != null) {
-								classes = assemblerClass.split(",");
-							}
-						} catch (IOException ignored) {
-							// we simply show no entries here
-						}
-
-						// show every entry class that can be loaded later on.
-						for (String clazz : classes) {
-							clazz = clazz.trim();
-
-							PackagedProgram program = null;
-							try {
-								program = new PackagedProgram(f, clazz, new String[0]);
-							} catch (Exception ignored) {
-								// ignore jar files which throw an error upon creating a PackagedProgram
-							}
-							if (program != null) {
-								gen.writeStartObject();
-								gen.writeStringField("name", clazz);
-								String desc = program.getDescription();
-								gen.writeStringField("description", desc == null ? "No description provided" : desc);
-								gen.writeEndObject();
-							}
-						}
-						gen.writeEndArray();
-						gen.writeEndObject();
-					}
-					gen.writeEndArray();
-					gen.writeEndObject();
-					gen.close();
-					return writer.toString();
-				}
-				catch (Exception e) {
-					throw new CompletionException(new FlinkException("Failed to fetch jar list.", e));
-				}
-			},
-			executor);
-
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarPlanHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarPlanHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarPlanHandler.java
deleted file mode 100644
index 81f3d0a..0000000
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarPlanHandler.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.webmonitor.handlers;
-
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobgraph.jsonplan.JsonPlanGenerator;
-import org.apache.flink.runtime.jobmaster.JobManagerGateway;
-import org.apache.flink.runtime.rest.handler.legacy.JsonFactory;
-
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
-
-import java.io.File;
-import java.io.StringWriter;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import java.util.concurrent.Executor;
-
-/**
- * This handler handles requests to fetch plan for a jar.
- */
-public class JarPlanHandler extends JarActionHandler {
-
-	static final String JAR_PLAN_REST_PATH = "/jars/:jarid/plan";
-
-	public JarPlanHandler(Executor executor, File jarDirectory) {
-		super(executor, jarDirectory);
-	}
-
-	@Override
-	public String[] getPaths() {
-		return new String[]{JAR_PLAN_REST_PATH};
-	}
-
-	@Override
-	public CompletableFuture<String> handleJsonRequest(Map<String, String> pathParams, Map<String, String> queryParams, JobManagerGateway jobManagerGateway) {
-		return CompletableFuture.supplyAsync(
-			() -> {
-				try {
-					JarActionHandlerConfig config = JarActionHandlerConfig.fromParams(pathParams, queryParams);
-					JobGraph graph = getJobGraphAndClassLoader(config).f0;
-					StringWriter writer = new StringWriter();
-					JsonGenerator gen = JsonFactory.JACKSON_FACTORY.createGenerator(writer);
-					gen.writeStartObject();
-					gen.writeFieldName("plan");
-					gen.writeRawValue(JsonPlanGenerator.generatePlan(graph));
-					gen.writeEndObject();
-					gen.close();
-					return writer.toString();
-				}
-				catch (Exception e) {
-					throw new CompletionException(e);
-				}
-			},
-			executor);
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandler.java
deleted file mode 100644
index dd200de..0000000
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandler.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.webmonitor.handlers;
-
-import org.apache.flink.api.common.time.Time;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.client.program.ProgramInvocationException;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.client.JobClient;
-import org.apache.flink.runtime.client.JobExecutionException;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobmaster.JobManagerGateway;
-import org.apache.flink.runtime.rest.handler.legacy.JsonFactory;
-import org.apache.flink.util.FlinkException;
-import org.apache.flink.util.Preconditions;
-
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
-
-import java.io.File;
-import java.io.StringWriter;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import java.util.concurrent.Executor;
-
-/**
- * This handler handles requests to fetch plan for a jar.
- */
-public class JarRunHandler extends JarActionHandler {
-
-	static final String JAR_RUN_REST_PATH = "/jars/:jarid/run";
-
-	private final Time timeout;
-	private final Configuration clientConfig;
-
-	public JarRunHandler(Executor executor, File jarDirectory, Time timeout, Configuration clientConfig) {
-		super(executor, jarDirectory);
-		this.timeout = Preconditions.checkNotNull(timeout);
-		this.clientConfig = Preconditions.checkNotNull(clientConfig);
-	}
-
-	@Override
-	public String[] getPaths() {
-		return new String[]{JAR_RUN_REST_PATH};
-	}
-
-	@Override
-	public CompletableFuture<String> handleJsonRequest(Map<String, String> pathParams, Map<String, String> queryParams, JobManagerGateway jobManagerGateway) {
-		return CompletableFuture.supplyAsync(
-			() -> {
-				try {
-					JarActionHandlerConfig config = JarActionHandlerConfig.fromParams(pathParams, queryParams);
-					Tuple2<JobGraph, ClassLoader> graph = getJobGraphAndClassLoader(config);
-
-					try {
-						JobClient.submitJobDetached(
-							jobManagerGateway,
-							clientConfig,
-							graph.f0,
-							timeout,
-							graph.f1);
-					} catch (JobExecutionException e) {
-						throw new ProgramInvocationException("Failed to submit the job to the job manager", e);
-					}
-
-					StringWriter writer = new StringWriter();
-					JsonGenerator gen = JsonFactory.JACKSON_FACTORY.createGenerator(writer);
-					gen.writeStartObject();
-					gen.writeStringField("jobid", graph.f0.getJobID().toString());
-					gen.writeEndObject();
-					gen.close();
-					return writer.toString();
-				} catch (Exception e) {
-					throw new CompletionException(new FlinkException("Could not run the jar.", e));
-				}
-			},
-			executor);
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadHandler.java
index 9a0bac4..66ae03b 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadHandler.java
@@ -18,67 +18,90 @@
 
 package org.apache.flink.runtime.webmonitor.handlers;
 
-import org.apache.flink.runtime.jobmaster.JobManagerGateway;
-import org.apache.flink.runtime.rest.handler.legacy.AbstractJsonRequestHandler;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.dispatcher.DispatcherGateway;
+import org.apache.flink.runtime.rest.handler.AbstractRestHandler;
+import org.apache.flink.runtime.rest.handler.HandlerRequest;
+import org.apache.flink.runtime.rest.handler.RestHandlerException;
+import org.apache.flink.runtime.rest.messages.EmptyMessageParameters;
+import org.apache.flink.runtime.rest.messages.FileUpload;
+import org.apache.flink.runtime.rest.messages.MessageHeaders;
+import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
 
-import java.io.File;
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus;
+
+import javax.annotation.Nonnull;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Map;
-import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
 import java.util.concurrent.Executor;
 
+import static java.util.Objects.requireNonNull;
+
 /**
- * Handles requests for uploading of jars.
+ * Handles .jar file uploads.
  */
-public class JarUploadHandler extends AbstractJsonRequestHandler {
+public class JarUploadHandler extends
+		AbstractRestHandler<DispatcherGateway, FileUpload, JarUploadResponseBody, EmptyMessageParameters> {
 
-	static final String JAR_UPLOAD_REST_PATH = "/jars/upload";
+	private final Path jarDir;
 
-	private final File jarDir;
+	private final Executor executor;
 
-	public JarUploadHandler(Executor executor, File jarDir) {
-		super(executor);
-		this.jarDir = jarDir;
+	public JarUploadHandler(
+			final CompletableFuture<String> localRestAddress,
+			final GatewayRetriever<? extends DispatcherGateway> leaderRetriever,
+			final Time timeout,
+			final Map<String, String> responseHeaders,
+			final MessageHeaders<FileUpload, JarUploadResponseBody, EmptyMessageParameters> messageHeaders,
+			final Path jarDir,
+			final Executor executor) {
+		super(localRestAddress, leaderRetriever, timeout, responseHeaders, messageHeaders);
+		this.jarDir = requireNonNull(jarDir);
+		this.executor = requireNonNull(executor);
 	}
 
 	@Override
-	public String[] getPaths() {
-		return new String[]{JAR_UPLOAD_REST_PATH};
-	}
+	protected CompletableFuture<JarUploadResponseBody> handleRequest(
+			@Nonnull final HandlerRequest<FileUpload, EmptyMessageParameters> request,
+			@Nonnull final DispatcherGateway gateway) throws RestHandlerException {
 
-	@Override
-	public CompletableFuture<String> handleJsonRequest(
-			Map<String, String> pathParams,
-			Map<String, String> queryParams,
-			JobManagerGateway jobManagerGateway) {
-
-		String tempFilePath = queryParams.get("filepath");
-		String filename = queryParams.get("filename");
-
-		return CompletableFuture.supplyAsync(
-			() -> {
-				File tempFile;
-				if (tempFilePath != null && (tempFile = new File(tempFilePath)).exists()) {
-					if (!tempFile.getName().endsWith(".jar")) {
-						//noinspection ResultOfMethodCallIgnored
-						tempFile.delete();
-						return "{\"error\": \"Only Jar files are allowed.\"}";
-					}
-
-					String filenameWithUUID = UUID.randomUUID() + "_" + filename;
-					File newFile = new File(jarDir, filenameWithUUID);
-					if (tempFile.renameTo(newFile)) {
-						// all went well
-						return "{\"status\": \"success\", \"filename\": \"" + filenameWithUUID + "\"}";
-					}
-					else {
-						//noinspection ResultOfMethodCallIgnored
-						tempFile.delete();
-					}
+		final FileUpload fileUpload = request.getRequestBody();
+		return CompletableFuture.supplyAsync(() -> {
+			if (!fileUpload.getPath().getFileName().toString().endsWith(".jar")) {
+				deleteUploadedFile(fileUpload);
+				throw new CompletionException(new RestHandlerException(
+					"Only Jar files are allowed.",
+					HttpResponseStatus.BAD_REQUEST));
+			} else {
+				final Path destination = jarDir.resolve(fileUpload.getPath().getFileName());
+				try {
+					Files.move(fileUpload.getPath(), destination);
+				} catch (IOException e) {
+					deleteUploadedFile(fileUpload);
+					throw new CompletionException(new RestHandlerException(
+						String.format("Could not move uploaded jar file [%s] to [%s].",
+							fileUpload.getPath(),
+							destination),
+						HttpResponseStatus.INTERNAL_SERVER_ERROR,
+						e));
 				}
+				return new JarUploadResponseBody(fileUpload.getPath()
+					.normalize()
+					.toString());
+			}
+		}, executor);
+	}
 
-				return "{\"error\": \"Failed to upload the file.\"}";
-			},
-			executor);
+	private void deleteUploadedFile(final FileUpload fileUpload) {
+		try {
+			Files.delete(fileUpload.getPath());
+		} catch (IOException e) {
+			log.error("Failed to delete file {}.", fileUpload.getPath(), e);
+		}
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadMessageHeaders.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadMessageHeaders.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadMessageHeaders.java
new file mode 100644
index 0000000..82c8d32
--- /dev/null
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadMessageHeaders.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers;
+
+import org.apache.flink.runtime.rest.HttpMethodWrapper;
+import org.apache.flink.runtime.rest.messages.EmptyMessageParameters;
+import org.apache.flink.runtime.rest.messages.FileUpload;
+import org.apache.flink.runtime.rest.messages.MessageHeaders;
+
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus;
+
+/**
+ * {@link MessageHeaders} for uploading jars.
+ */
+public final class JarUploadMessageHeaders implements MessageHeaders<FileUpload, JarUploadResponseBody, EmptyMessageParameters> {
+
+	@Override
+	public Class<JarUploadResponseBody> getResponseClass() {
+		return JarUploadResponseBody.class;
+	}
+
+	@Override
+	public HttpResponseStatus getResponseStatusCode() {
+		return HttpResponseStatus.OK;
+	}
+
+	@Override
+	public Class<FileUpload> getRequestClass() {
+		return FileUpload.class;
+	}
+
+	@Override
+	public EmptyMessageParameters getUnresolvedMessageParameters() {
+		return EmptyMessageParameters.getInstance();
+	}
+
+	@Override
+	public HttpMethodWrapper getHttpMethod() {
+		return HttpMethodWrapper.POST;
+	}
+
+	@Override
+	public String getTargetRestEndpointURL() {
+		return "/jars/upload";
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadResponseBody.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadResponseBody.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadResponseBody.java
new file mode 100644
index 0000000..dbd2cfe
--- /dev/null
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadResponseBody.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers;
+
+import org.apache.flink.runtime.rest.messages.ResponseBody;
+
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * {@link ResponseBody} for {@link JarUploadHandler}.
+ */
+public class JarUploadResponseBody implements ResponseBody {
+
+	private static final String FIELD_NAME_FILENAME = "filename";
+
+	private static final String FIELD_NAME_STATUS = "status";
+
+	@JsonProperty(FIELD_NAME_STATUS)
+	private final UploadStatus status = UploadStatus.success;
+
+	@JsonProperty(FIELD_NAME_FILENAME)
+	private final String filename;
+
+	@JsonCreator
+	public JarUploadResponseBody(@JsonProperty(FIELD_NAME_FILENAME) final String filename) {
+		this.filename = requireNonNull(filename);
+	}
+
+	public UploadStatus getStatus() {
+		return status;
+	}
+
+	public String getFilename() {
+		return filename;
+	}
+
+	enum UploadStatus {
+		success
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarAccessDeniedHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarAccessDeniedHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarAccessDeniedHandler.java
new file mode 100644
index 0000000..1600a3b
--- /dev/null
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarAccessDeniedHandler.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers.legacy;
+
+import org.apache.flink.runtime.jobmaster.JobManagerGateway;
+import org.apache.flink.runtime.rest.handler.legacy.AbstractJsonRequestHandler;
+
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+
+/**
+ * Handler to deny access to jar-related REST calls.
+ */
+public class JarAccessDeniedHandler extends AbstractJsonRequestHandler {
+
+	private static final String ERROR_MESSAGE = "{\"error\": \"Web submission interface is not " +
+			"available for this cluster. To enable it, set the configuration key ' jobmanager.web.submit.enable.'\"}";
+
+	public JarAccessDeniedHandler(Executor executor) {
+		super(executor);
+	}
+
+	@Override
+	public String[] getPaths() {
+		return new String[]{
+			JarListHandler.JAR_LIST_REST_PATH,
+			JarPlanHandler.JAR_PLAN_REST_PATH,
+			JarRunHandler.JAR_RUN_REST_PATH,
+			JarUploadHandler.JAR_UPLOAD_REST_PATH,
+			JarDeleteHandler.JAR_DELETE_REST_PATH
+		};
+	}
+
+	@Override
+	public CompletableFuture<String> handleJsonRequest(Map<String, String> pathParams, Map<String, String> queryParams, JobManagerGateway jobManagerGateway) {
+		return CompletableFuture.completedFuture(ERROR_MESSAGE);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarActionHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarActionHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarActionHandler.java
new file mode 100644
index 0000000..de48e36
--- /dev/null
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarActionHandler.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers.legacy;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.client.program.PackagedProgram;
+import org.apache.flink.client.program.ProgramInvocationException;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.costs.DefaultCostEstimator;
+import org.apache.flink.optimizer.plan.FlinkPlan;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.StreamingPlan;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+import org.apache.flink.runtime.rest.handler.legacy.AbstractJsonRequestHandler;
+import org.apache.flink.runtime.rest.handler.legacy.JsonFactory;
+import org.apache.flink.runtime.webmonitor.WebRuntimeMonitor;
+import org.apache.flink.util.ExceptionUtils;
+
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executor;
+
+/**
+ * Abstract handler for fetching plan for a jar or running a jar.
+ */
+public abstract class JarActionHandler extends AbstractJsonRequestHandler {
+
+	private final File jarDir;
+
+	public JarActionHandler(Executor executor, File jarDirectory) {
+		super(executor);
+		jarDir = jarDirectory;
+	}
+
+	protected Tuple2<JobGraph, ClassLoader> getJobGraphAndClassLoader(JarActionHandlerConfig config) throws Exception {
+		// generate the graph
+		JobGraph graph = null;
+
+		if (!jarDir.exists()) {
+			WebRuntimeMonitor.logExternalUploadDirDeletion(jarDir);
+			try {
+				WebRuntimeMonitor.checkAndCreateUploadDir(jarDir);
+			} catch (IOException ioe) {
+				// the following code will throw an exception since the jar can't be found
+			}
+		}
+
+		PackagedProgram program = new PackagedProgram(
+				new File(jarDir, config.getJarFile()),
+				config.getEntryClass(),
+				config.getProgramArgs());
+		ClassLoader classLoader = program.getUserCodeClassLoader();
+
+		Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration());
+		FlinkPlan plan = ClusterClient.getOptimizedPlan(optimizer, program, config.getParallelism());
+
+		if (plan instanceof StreamingPlan) {
+			graph = ((StreamingPlan) plan).getJobGraph();
+		} else if (plan instanceof OptimizedPlan) {
+			graph = new JobGraphGenerator().compileJobGraph((OptimizedPlan) plan);
+		}
+		if (graph == null) {
+			throw new CompilerException("A valid job graph couldn't be generated for the jar.");
+		}
+
+		// Set the savepoint settings
+		graph.setSavepointRestoreSettings(config.getSavepointRestoreSettings());
+
+		for (URL jar : program.getAllLibraries()) {
+			try {
+				graph.addJar(new Path(jar.toURI()));
+			}
+			catch (URISyntaxException e) {
+				throw new ProgramInvocationException("Invalid jar path. Unexpected error. :(");
+			}
+		}
+		return Tuple2.of(graph, classLoader);
+	}
+
+	protected String sendError(Exception e) throws Exception {
+		StringWriter writer = new StringWriter();
+		JsonGenerator gen = JsonFactory.JACKSON_FACTORY.createGenerator(writer);
+
+		gen.writeStartObject();
+		gen.writeStringField("error", ExceptionUtils.stringifyException(e));
+		gen.writeEndObject();
+		gen.close();
+
+		return writer.toString();
+	}
+
+	/**
+	 * Wrapper for all configuration that is parsed from query and path args.
+	 */
+	@VisibleForTesting
+	static class JarActionHandlerConfig {
+
+		private final String jarFile;
+		private final String[] programArgs;
+		private final String entryClass;
+		private final int parallelism;
+		private final SavepointRestoreSettings savepointRestoreSettings;
+
+		JarActionHandlerConfig(
+				String jarFile,
+				String[] programArgs,
+				String entryClass,
+				int parallelism,
+				SavepointRestoreSettings savepointRestoreSettings) {
+
+			this.jarFile = jarFile;
+			this.programArgs = programArgs;
+			this.entryClass = entryClass;
+			this.parallelism = parallelism;
+			this.savepointRestoreSettings = savepointRestoreSettings;
+		}
+
+		public String getJarFile() {
+			return jarFile;
+		}
+
+		public String[] getProgramArgs() {
+			return programArgs;
+		}
+
+		public String getEntryClass() {
+			return entryClass;
+		}
+
+		public int getParallelism() {
+			return parallelism;
+		}
+
+		public SavepointRestoreSettings getSavepointRestoreSettings() {
+			return savepointRestoreSettings;
+		}
+
+		public static JarActionHandlerConfig fromParams(Map<String, String> pathParams, Map<String, String> queryParams) {
+			// Jar file
+			String jarFile = pathParams.get("jarid");
+			if (jarFile == null) {
+				throw new IllegalArgumentException("No jarid was provided.");
+			}
+
+			// Program args
+			String[] programArgs = new String[0];
+			String programArgsOpt = queryParams.get("program-args");
+			if (programArgsOpt != null && !programArgsOpt.equals("")) {
+				List<String> args = tokenizeArguments(programArgsOpt);
+				programArgs = args.toArray(new String[args.size()]);
+			}
+
+			// Entry class
+			String entryClass = null;
+			String entryClassOpt = queryParams.get("entry-class");
+			if (entryClassOpt != null && !entryClassOpt.equals("")) {
+				entryClass = entryClassOpt;
+			}
+
+			// Parallelism
+			int parallelism = 1;
+			String parallelismOpt = queryParams.get("parallelism");
+			if (parallelismOpt != null && !parallelismOpt.equals("")) {
+				parallelism = Integer.parseInt(parallelismOpt);
+				if (parallelism < 1) {
+					throw new IllegalArgumentException("Parallelism must be a positive number.");
+				}
+			}
+
+			// Savepoint restore settings
+			SavepointRestoreSettings savepointSettings = SavepointRestoreSettings.none();
+			String savepointPath = queryParams.get("savepointPath");
+			if (savepointPath != null && !savepointPath.equals("")) {
+				String allowNonRestoredOpt = queryParams.get("allowNonRestoredState");
+				boolean allowNonRestoredState = allowNonRestoredOpt != null && allowNonRestoredOpt.equals("true");
+				savepointSettings = SavepointRestoreSettings.forPath(savepointPath, allowNonRestoredState);
+			}
+
+			return new JarActionHandlerConfig(jarFile, programArgs, entryClass, parallelism, savepointSettings);
+		}
+
+		/**
+		 * Utility method that takes the given arguments, splits them at the whitespaces (space and tab) and
+		 * turns them into an array of Strings. Other than the <tt>StringTokenizer</tt>, this method
+		 * takes care of quotes, such that quoted passages end up being one string.
+		 *
+		 * @param args
+		 *        The string to be split.
+		 * @return The array of split strings.
+		 */
+		private static List<String> tokenizeArguments(String args) {
+			List<String> list = new ArrayList<String>();
+			StringBuilder curr = new StringBuilder();
+
+			int pos = 0;
+			boolean quoted = false;
+
+			while (pos < args.length()) {
+				char c = args.charAt(pos);
+				if ((c == ' ' || c == '\t') && !quoted) {
+					if (curr.length() > 0) {
+						list.add(curr.toString());
+						curr.setLength(0);
+					}
+				} else if (c == '"') {
+					quoted = !quoted;
+				} else {
+					curr.append(c);
+				}
+
+				pos++;
+			}
+
+			if (quoted) {
+				throw new IllegalArgumentException("Unterminated quoted string.");
+			}
+
+			if (curr.length() > 0) {
+				list.add(curr.toString());
+			}
+
+			return list;
+		}
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarDeleteHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarDeleteHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarDeleteHandler.java
new file mode 100644
index 0000000..315a73f
--- /dev/null
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarDeleteHandler.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers.legacy;
+
+import org.apache.flink.runtime.jobmaster.JobManagerGateway;
+import org.apache.flink.runtime.rest.handler.legacy.AbstractJsonRequestHandler;
+import org.apache.flink.runtime.rest.handler.legacy.JsonFactory;
+import org.apache.flink.runtime.webmonitor.WebRuntimeMonitor;
+import org.apache.flink.util.FlinkException;
+
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.Executor;
+
+/**
+ * Handles requests for deletion of jars.
+ */
+public class JarDeleteHandler extends AbstractJsonRequestHandler {
+
+	static final String JAR_DELETE_REST_PATH = "/jars/:jarid";
+
+	private final File jarDir;
+
+	public JarDeleteHandler(Executor executor, File jarDirectory) {
+		super(executor);
+		jarDir = jarDirectory;
+	}
+
+	@Override
+	public String[] getPaths() {
+		return new String[]{JAR_DELETE_REST_PATH};
+	}
+
+	@Override
+	public CompletableFuture<String> handleJsonRequest(Map<String, String> pathParams, Map<String, String> queryParams, JobManagerGateway jobManagerGateway) {
+		final String file = pathParams.get("jarid");
+		return CompletableFuture.supplyAsync(
+			() -> {
+				try {
+					File[] list = jarDir.listFiles(new FilenameFilter() {
+						@Override
+						public boolean accept(File dir, String name) {
+							return name.equals(file);
+						}
+					});
+
+					if (list == null) {
+						WebRuntimeMonitor.logExternalUploadDirDeletion(jarDir);
+						try {
+							WebRuntimeMonitor.checkAndCreateUploadDir(jarDir);
+						} catch (IOException ioe) {
+							// entire directory doesn't exist anymore, continue as if deletion succeeded
+						}
+						list = new File[0];
+					}
+
+					boolean success = false;
+					for (File f: list) {
+						// although next to impossible for multiple files, we still delete them.
+						success = success || f.delete();
+					}
+					StringWriter writer = new StringWriter();
+					JsonGenerator gen = JsonFactory.JACKSON_FACTORY.createGenerator(writer);
+					gen.writeStartObject();
+					if (!success) {
+						// this seems to always fail on Windows.
+						gen.writeStringField("error", "The requested jar couldn't be deleted. Please try again.");
+					}
+					gen.writeEndObject();
+					gen.close();
+					return writer.toString();
+				}
+				catch (Exception e) {
+					throw new CompletionException(new FlinkException("Failed to delete jar id " + pathParams.get("jarid") + '.', e));
+				}
+			},
+			executor);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarListHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarListHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarListHandler.java
new file mode 100644
index 0000000..77f7d4f
--- /dev/null
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarListHandler.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers.legacy;
+
+import org.apache.flink.client.program.PackagedProgram;
+import org.apache.flink.runtime.jobmaster.JobManagerGateway;
+import org.apache.flink.runtime.rest.handler.legacy.AbstractJsonRequestHandler;
+import org.apache.flink.runtime.rest.handler.legacy.JsonFactory;
+import org.apache.flink.runtime.webmonitor.RuntimeMonitorHandler;
+import org.apache.flink.runtime.webmonitor.WebRuntimeMonitor;
+import org.apache.flink.util.FlinkException;
+
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.Executor;
+import java.util.jar.JarFile;
+import java.util.jar.Manifest;
+
+/**
+ * Handle request for listing uploaded jars.
+ */
+public class JarListHandler extends AbstractJsonRequestHandler {
+
+	static final String JAR_LIST_REST_PATH = "/jars";
+
+	private final File jarDir;
+
+	public  JarListHandler(Executor executor, File jarDirectory) {
+		super(executor);
+		jarDir = jarDirectory;
+	}
+
+	@Override
+	public String[] getPaths() {
+		return new String[]{JAR_LIST_REST_PATH};
+	}
+
+	@Override
+	public CompletableFuture<String> handleJsonRequest(Map<String, String> pathParams, Map<String, String> queryParams, JobManagerGateway jobManagerGateway) {
+		return CompletableFuture.supplyAsync(
+			() -> {
+				try {
+					StringWriter writer = new StringWriter();
+					JsonGenerator gen = JsonFactory.JACKSON_FACTORY.createGenerator(writer);
+
+					gen.writeStartObject();
+					gen.writeStringField("address", queryParams.get(RuntimeMonitorHandler.WEB_MONITOR_ADDRESS_KEY));
+					gen.writeArrayFieldStart("files");
+
+					File[] list = jarDir.listFiles(new FilenameFilter() {
+						@Override
+						public boolean accept(File dir, String name) {
+							return name.endsWith(".jar");
+						}
+					});
+
+					if (list == null) {
+						WebRuntimeMonitor.logExternalUploadDirDeletion(jarDir);
+						try {
+							WebRuntimeMonitor.checkAndCreateUploadDir(jarDir);
+						} catch (IOException ioe) {
+							// re-throwing an exception here breaks the UI
+						}
+						list = new File[0];
+					}
+
+					// last modified ascending order
+					Arrays.sort(list, (f1, f2) -> Long.compare(f2.lastModified(), f1.lastModified()));
+
+					for (File f : list) {
+						// separate the uuid and the name parts.
+						String id = f.getName();
+
+						int startIndex = id.indexOf("_");
+						if (startIndex < 0) {
+							continue;
+						}
+						String name = id.substring(startIndex + 1);
+						if (name.length() < 5 || !name.endsWith(".jar")) {
+							continue;
+						}
+
+						gen.writeStartObject();
+						gen.writeStringField("id", id);
+						gen.writeStringField("name", name);
+						gen.writeNumberField("uploaded", f.lastModified());
+						gen.writeArrayFieldStart("entry");
+
+						String[] classes = new String[0];
+						try (JarFile jar = new JarFile(f)) {
+							Manifest manifest = jar.getManifest();
+							String assemblerClass = null;
+
+							if (manifest != null) {
+								assemblerClass = manifest.getMainAttributes().getValue(PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS);
+								if (assemblerClass == null) {
+									assemblerClass = manifest.getMainAttributes().getValue(PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS);
+								}
+							}
+							if (assemblerClass != null) {
+								classes = assemblerClass.split(",");
+							}
+						} catch (IOException ignored) {
+							// we simply show no entries here
+						}
+
+						// show every entry class that can be loaded later on.
+						for (String clazz : classes) {
+							clazz = clazz.trim();
+
+							PackagedProgram program = null;
+							try {
+								program = new PackagedProgram(f, clazz, new String[0]);
+							} catch (Exception ignored) {
+								// ignore jar files which throw an error upon creating a PackagedProgram
+							}
+							if (program != null) {
+								gen.writeStartObject();
+								gen.writeStringField("name", clazz);
+								String desc = program.getDescription();
+								gen.writeStringField("description", desc == null ? "No description provided" : desc);
+								gen.writeEndObject();
+							}
+						}
+						gen.writeEndArray();
+						gen.writeEndObject();
+					}
+					gen.writeEndArray();
+					gen.writeEndObject();
+					gen.close();
+					return writer.toString();
+				}
+				catch (Exception e) {
+					throw new CompletionException(new FlinkException("Failed to fetch jar list.", e));
+				}
+			},
+			executor);
+
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarPlanHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarPlanHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarPlanHandler.java
new file mode 100644
index 0000000..3069533
--- /dev/null
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarPlanHandler.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers.legacy;
+
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.jsonplan.JsonPlanGenerator;
+import org.apache.flink.runtime.jobmaster.JobManagerGateway;
+import org.apache.flink.runtime.rest.handler.legacy.JsonFactory;
+
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
+
+import java.io.File;
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.Executor;
+
+/**
+ * This handler handles requests to fetch plan for a jar.
+ */
+public class JarPlanHandler extends JarActionHandler {
+
+	static final String JAR_PLAN_REST_PATH = "/jars/:jarid/plan";
+
+	public JarPlanHandler(Executor executor, File jarDirectory) {
+		super(executor, jarDirectory);
+	}
+
+	@Override
+	public String[] getPaths() {
+		return new String[]{JAR_PLAN_REST_PATH};
+	}
+
+	@Override
+	public CompletableFuture<String> handleJsonRequest(Map<String, String> pathParams, Map<String, String> queryParams, JobManagerGateway jobManagerGateway) {
+		return CompletableFuture.supplyAsync(
+			() -> {
+				try {
+					JarActionHandlerConfig config = JarActionHandlerConfig.fromParams(pathParams, queryParams);
+					JobGraph graph = getJobGraphAndClassLoader(config).f0;
+					StringWriter writer = new StringWriter();
+					JsonGenerator gen = JsonFactory.JACKSON_FACTORY.createGenerator(writer);
+					gen.writeStartObject();
+					gen.writeFieldName("plan");
+					gen.writeRawValue(JsonPlanGenerator.generatePlan(graph));
+					gen.writeEndObject();
+					gen.close();
+					return writer.toString();
+				}
+				catch (Exception e) {
+					throw new CompletionException(e);
+				}
+			},
+			executor);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarRunHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarRunHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarRunHandler.java
new file mode 100644
index 0000000..72caa3c
--- /dev/null
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarRunHandler.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers.legacy;
+
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.client.program.ProgramInvocationException;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.client.JobClient;
+import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobmaster.JobManagerGateway;
+import org.apache.flink.runtime.rest.handler.legacy.JsonFactory;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.Preconditions;
+
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
+
+import java.io.File;
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.Executor;
+
+/**
+ * This handler handles requests to fetch plan for a jar.
+ */
+public class JarRunHandler extends JarActionHandler {
+
+	static final String JAR_RUN_REST_PATH = "/jars/:jarid/run";
+
+	private final Time timeout;
+	private final Configuration clientConfig;
+
+	public JarRunHandler(Executor executor, File jarDirectory, Time timeout, Configuration clientConfig) {
+		super(executor, jarDirectory);
+		this.timeout = Preconditions.checkNotNull(timeout);
+		this.clientConfig = Preconditions.checkNotNull(clientConfig);
+	}
+
+	@Override
+	public String[] getPaths() {
+		return new String[]{JAR_RUN_REST_PATH};
+	}
+
+	@Override
+	public CompletableFuture<String> handleJsonRequest(Map<String, String> pathParams, Map<String, String> queryParams, JobManagerGateway jobManagerGateway) {
+		return CompletableFuture.supplyAsync(
+			() -> {
+				try {
+					JarActionHandlerConfig config = JarActionHandlerConfig.fromParams(pathParams, queryParams);
+					Tuple2<JobGraph, ClassLoader> graph = getJobGraphAndClassLoader(config);
+
+					try {
+						JobClient.submitJobDetached(
+							jobManagerGateway,
+							clientConfig,
+							graph.f0,
+							timeout,
+							graph.f1);
+					} catch (JobExecutionException e) {
+						throw new ProgramInvocationException("Failed to submit the job to the job manager", e);
+					}
+
+					StringWriter writer = new StringWriter();
+					JsonGenerator gen = JsonFactory.JACKSON_FACTORY.createGenerator(writer);
+					gen.writeStartObject();
+					gen.writeStringField("jobid", graph.f0.getJobID().toString());
+					gen.writeEndObject();
+					gen.close();
+					return writer.toString();
+				} catch (Exception e) {
+					throw new CompletionException(new FlinkException("Could not run the jar.", e));
+				}
+			},
+			executor);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarUploadHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarUploadHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarUploadHandler.java
new file mode 100644
index 0000000..bd9be02
--- /dev/null
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarUploadHandler.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers.legacy;
+
+import org.apache.flink.runtime.jobmaster.JobManagerGateway;
+import org.apache.flink.runtime.rest.handler.legacy.AbstractJsonRequestHandler;
+
+import java.io.File;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+
+/**
+ * Handles requests for uploading of jars.
+ */
+public class JarUploadHandler extends AbstractJsonRequestHandler {
+
+	static final String JAR_UPLOAD_REST_PATH = "/jars/upload";
+
+	private final File jarDir;
+
+	public JarUploadHandler(Executor executor, File jarDir) {
+		super(executor);
+		this.jarDir = jarDir;
+	}
+
+	@Override
+	public String[] getPaths() {
+		return new String[]{JAR_UPLOAD_REST_PATH};
+	}
+
+	@Override
+	public CompletableFuture<String> handleJsonRequest(
+			Map<String, String> pathParams,
+			Map<String, String> queryParams,
+			JobManagerGateway jobManagerGateway) {
+
+		String tempFilePath = queryParams.get("filepath");
+		String filename = queryParams.get("filename");
+
+		return CompletableFuture.supplyAsync(
+			() -> {
+				File tempFile;
+				if (tempFilePath != null && (tempFile = new File(tempFilePath)).exists()) {
+					if (!tempFile.getName().endsWith(".jar")) {
+						//noinspection ResultOfMethodCallIgnored
+						tempFile.delete();
+						return "{\"error\": \"Only Jar files are allowed.\"}";
+					}
+
+					String filenameWithUUID = UUID.randomUUID() + "_" + filename;
+					File newFile = new File(jarDir, filenameWithUUID);
+					if (tempFile.renameTo(newFile)) {
+						// all went well
+						return "{\"status\": \"success\", \"filename\": \"" + filenameWithUUID + "\"}";
+					}
+					else {
+						//noinspection ResultOfMethodCallIgnored
+						tempFile.delete();
+					}
+				}
+
+				return "{\"error\": \"Failed to upload the file.\"}";
+			},
+			executor);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebMonitorUtilsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebMonitorUtilsTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebMonitorUtilsTest.java
index f2a39c5..772584a 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebMonitorUtilsTest.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebMonitorUtilsTest.java
@@ -18,11 +18,22 @@
 
 package org.apache.flink.runtime.webmonitor;
 
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.runtime.webmonitor.handlers.JarUploadHandler;
 import org.apache.flink.runtime.webmonitor.history.JsonArchivist;
 
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.concurrent.CompletableFuture;
+
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertThat;
+
 /**
  * Tests for the WebMonitorUtils.
  */
@@ -38,4 +49,18 @@ public class WebMonitorUtilsTest {
 			Assert.assertSame(direct[x].getClass(), reflected[x].getClass());
 		}
 	}
+
+	/**
+	 * Tests dynamically loading of handlers such as {@link JarUploadHandler}.
+	 */
+	@Test
+	public void testTryLoadJarHandlers() {
+		assertThat(WebMonitorUtils.tryLoadJarHandlers(
+			CompletableFuture::new,
+			CompletableFuture.completedFuture("localhost:12345"),
+			Time.seconds(10),
+			Collections.emptyMap(),
+			Paths.get("/tmp"),
+			Executors.directExecutor()), not(empty()));
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarAccessDeniedHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarAccessDeniedHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarAccessDeniedHandlerTest.java
deleted file mode 100644
index 97091cf..0000000
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarAccessDeniedHandlerTest.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.webmonitor.handlers;
-
-import org.apache.flink.runtime.concurrent.Executors;
-
-import org.apache.flink.shaded.guava18.com.google.common.collect.Lists;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.List;
-
-/**
- * Tests for the JarAccessDeniedHandler.
- */
-public class JarAccessDeniedHandlerTest {
-	@Test
-	public void testGetPaths() {
-		JarAccessDeniedHandler handler = new JarAccessDeniedHandler(Executors.directExecutor());
-		String[] paths = handler.getPaths();
-		Assert.assertEquals(5, paths.length);
-		List<String> pathsList = Lists.newArrayList(paths);
-		Assert.assertTrue(pathsList.contains("/jars"));
-		Assert.assertTrue(pathsList.contains("/jars/upload"));
-		Assert.assertTrue(pathsList.contains("/jars/:jarid"));
-		Assert.assertTrue(pathsList.contains("/jars/:jarid/plan"));
-		Assert.assertTrue(pathsList.contains("/jars/:jarid/run"));
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarActionHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarActionHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarActionHandlerTest.java
deleted file mode 100644
index eb0f6b3..0000000
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarActionHandlerTest.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.webmonitor.handlers;
-
-import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Tests for the JarActionHandler.
- */
-public class JarActionHandlerTest {
-
-	/**
-	 * Test that the savepoint settings are correctly parsed.
-	 */
-	@Test
-	public void testSavepointRestoreSettings() throws Exception {
-		Map<String, String> pathParams = new HashMap<>();
-		pathParams.put("jarid", "required"); // required
-
-		// the following should be ignored, because they are parsed from the query params
-		pathParams.put("savepointPath", "ignored");
-		pathParams.put("allowNonRestoredState", "ignored");
-
-		Map<String, String> queryParams = new HashMap<>(); // <-- everything goes here
-
-		// Nothing configured
-		JarActionHandler.JarActionHandlerConfig config = JarActionHandler.JarActionHandlerConfig.fromParams(pathParams, queryParams);
-		assertEquals(SavepointRestoreSettings.none(), config.getSavepointRestoreSettings());
-
-		// Set path
-		queryParams.put("savepointPath", "the-savepoint-path");
-		queryParams.put("allowNonRestoredState", "");
-
-		SavepointRestoreSettings expected = SavepointRestoreSettings.forPath("the-savepoint-path", false);
-
-		config = JarActionHandler.JarActionHandlerConfig.fromParams(pathParams, queryParams);
-		assertEquals(expected, config.getSavepointRestoreSettings());
-
-		// Set flag
-		queryParams.put("allowNonRestoredState", "true");
-
-		expected = SavepointRestoreSettings.forPath("the-savepoint-path", true);
-		config = JarActionHandler.JarActionHandlerConfig.fromParams(pathParams, queryParams);
-		assertEquals(expected, config.getSavepointRestoreSettings());
-	}
-
-	/**
-	 * Tests that empty String params are handled ignored.
-	 */
-	@Test
-	public void testEmptyStringParams() throws Exception {
-		Map<String, String> pathParams = new HashMap<>();
-		pathParams.put("jarid", "required"); // required
-		Map<String, String> queryParams = new HashMap<>();
-
-		queryParams.put("program-args", "");
-		queryParams.put("entry-class", "");
-		queryParams.put("parallelism", "");
-		queryParams.put("savepointPath", "");
-		queryParams.put("allowNonRestoredState", "");
-
-		// Nothing configured
-		JarActionHandler.JarActionHandlerConfig config = JarActionHandler.JarActionHandlerConfig.fromParams(pathParams, queryParams);
-
-		assertEquals(0, config.getProgramArgs().length);
-		Assert.assertNull(config.getEntryClass());
-		assertEquals(1, config.getParallelism());
-		assertEquals(SavepointRestoreSettings.none(), config.getSavepointRestoreSettings());
-	}
-}


[19/22] flink git commit: [FLINK-8643] [flip6] Use JobManagerOptions#SLOT_REQUEST_TIMEOUT in ExecutionGraph

Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerSlotSharingTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerSlotSharingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerSlotSharingTest.java
index 025c795..1cc9301 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerSlotSharingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerSlotSharingTest.java
@@ -68,10 +68,10 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			testingSlotProvider.addTaskManager(2);
 			
 			// schedule 4 tasks from the first vertex group
-			LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
 			assertNotNull(s1);
 			assertNotNull(s2);
@@ -82,7 +82,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			
 			// we cannot schedule another task from the first vertex group
 			try {
-				testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+				testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 				fail("Scheduler accepted too many tasks at the same time");
 			}
 			catch (ExecutionException e) {
@@ -96,7 +96,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			s3.releaseSlot();
 			
 			// allocate another slot from that group
-			LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			assertNotNull(s5);
 			
 			// release all old slots
@@ -104,9 +104,9 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			s2.releaseSlot();
 			s4.releaseSlot();
 			
-			LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 5, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 6, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s8 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 7, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 5, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 6, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s8 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 7, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
 			assertNotNull(s6);
 			assertNotNull(s7);
@@ -151,10 +151,10 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 		testingSlotProvider.addTaskManager(2);
 
 		// schedule 4 tasks from the first vertex group
-		LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-		LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-		LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-		LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+		LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 		assertNotNull(s1);
 		assertNotNull(s2);
@@ -165,7 +165,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 
 		// we cannot schedule another task from the first vertex group
 		try {
-			testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			fail("Scheduler accepted too many tasks at the same time");
 		}
 		catch (ExecutionException e) {
@@ -176,10 +176,10 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 		}
 
 		// schedule some tasks from the second ID group
-		LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-		LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-		LogicalSlot s3_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-		LogicalSlot s4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+		LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s3_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 		assertNotNull(s1_2);
 		assertNotNull(s2_2);
@@ -188,7 +188,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 
 		// we cannot schedule another task from the second vertex group
 		try {
-			testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			fail("Scheduler accepted too many tasks at the same time");
 		}
 		catch (ExecutionException e) {
@@ -209,7 +209,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 
 		// we can still not schedule anything from the second group of vertices
 		try {
-			testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			fail("Scheduler accepted too many tasks at the same time");
 		}
 		catch (ExecutionException e) {
@@ -220,7 +220,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 		}
 
 		// we can schedule something from the first vertex group
-		LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+		LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 		assertNotNull(s5);
 
 		assertEquals(4, testingSlotProvider.getNumberOfSlots(sharingGroup));
@@ -230,7 +230,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 
 		// now we release a slot from the second vertex group and schedule another task from that group
 		s2_2.releaseSlot();
-		LogicalSlot s5_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+		LogicalSlot s5_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 		assertNotNull(s5_2);
 
 		// release all slots
@@ -265,10 +265,10 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			testingSlotProvider.addTaskManager(2);
 
 			// schedule 4 tasks from the first vertex group
-			LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
 			assertEquals(4, testingSlotProvider.getNumberOfSlots(sharingGroup));
 			assertEquals(0, testingSlotProvider.getNumberOfAvailableSlotsForGroup(sharingGroup, jid1));
@@ -284,10 +284,10 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			assertEquals(0, testingSlotProvider.getNumberOfAvailableSlotsForGroup(sharingGroup, jid2));
 			
 			// schedule some tasks from the second ID group
-			LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s3_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s3_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 			assertEquals(4, testingSlotProvider.getNumberOfSlots(sharingGroup));
 			assertEquals(4, testingSlotProvider.getNumberOfAvailableSlotsForGroup(sharingGroup, jid1));
@@ -329,10 +329,10 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			testingSlotProvider.addTaskManager(2);
 
 			// schedule 4 tasks from the first vertex group
-			LogicalSlot s1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s3_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s4_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s3_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s4_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
 			assertNotNull(s1_1);
 			assertNotNull(s2_1);
@@ -342,10 +342,10 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			assertTrue(areAllDistinct(s1_1, s2_1, s3_1, s4_1));
 			
 			// schedule 4 tasks from the second vertex group
-			LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s3_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s3_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
 			assertNotNull(s1_2);
 			assertNotNull(s2_2);
@@ -355,10 +355,10 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			assertTrue(areAllDistinct(s1_2, s2_2, s3_2, s4_2));
 			
 			// schedule 4 tasks from the third vertex group
-			LogicalSlot s1_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s2_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s3_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s4_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s1_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s3_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s4_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
 			assertNotNull(s1_3);
 			assertNotNull(s2_3);
@@ -370,7 +370,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			
 			// we cannot schedule another task from the second vertex group
 			try {
-				testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+				testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 				fail("Scheduler accepted too many tasks at the same time");
 			}
 			catch (ExecutionException e) {
@@ -386,9 +386,9 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			s3_2.releaseSlot();
 			s4_2.releaseSlot();
 			
-			LogicalSlot s5_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 5, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s6_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 6, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s7_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 7, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s5_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 5, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s6_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 6, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s7_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 7, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
 			assertNotNull(s5_2);
 			assertNotNull(s6_2);
@@ -438,9 +438,9 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			testingSlotProvider.addTaskManager(2);
 
 			// schedule 1 tasks from the first vertex group and 2 from the second
-			LogicalSlot s1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 2, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 2, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 2, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 2, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 2, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 2, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
 			assertNotNull(s1_1);
 			assertNotNull(s2_1);
@@ -456,7 +456,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			
 			
 			// this should free one slot so we can allocate one non-shared
-			LogicalSlot sx = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, null)), false, Collections.emptyList()).get();
+			LogicalSlot sx = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			assertNotNull(sx);
 			
 			assertEquals(1, testingSlotProvider.getNumberOfSlots(sharingGroup));
@@ -490,28 +490,28 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			testingSlotProvider.addTaskManager(2);
 
 			// schedule some individual vertices
-			LogicalSlot sA2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidA, 1, 2, null)), false, Collections.emptyList()).get();
-			LogicalSlot sA1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidA, 0, 2, null)), false, Collections.emptyList()).get();
+			LogicalSlot sA2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidA, 1, 2, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot sA1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidA, 0, 2, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			assertNotNull(sA1);
 			assertNotNull(sA2);
 			
 			// schedule some vertices in the sharing group
-			LogicalSlot s1_0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s2_0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s1_0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2_0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			assertNotNull(s1_0);
 			assertNotNull(s1_1);
 			assertNotNull(s2_0);
 			assertNotNull(s2_1);
 			
 			// schedule another isolated vertex
-			LogicalSlot sB1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 1, 3, null)), false, Collections.emptyList()).get();
+			LogicalSlot sB1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 1, 3, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			assertNotNull(sB1);
 			
 			// should not be able to schedule more vertices
 			try {
-				testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+				testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 				fail("Scheduler accepted too many tasks at the same time");
 			}
 			catch (ExecutionException e) {
@@ -522,7 +522,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			}
 			
 			try {
-				testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+				testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 				fail("Scheduler accepted too many tasks at the same time");
 			}
 			catch (ExecutionException e) {
@@ -533,7 +533,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			}
 			
 			try {
-				testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 0, 3, null)), false, Collections.emptyList()).get();
+				testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 0, 3, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 				fail("Scheduler accepted too many tasks at the same time");
 			}
 			catch (ExecutionException e) {
@@ -544,7 +544,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			}
 			
 			try {
-				testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 0, 1, null)), false, Collections.emptyList()).get();
+				testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 0, 1, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 				fail("Scheduler accepted too many tasks at the same time");
 			}
 			catch (ExecutionException e) {
@@ -557,8 +557,8 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			// release some isolated task and check that the sharing group may grow
 			sA1.releaseSlot();
 			
-			LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			assertNotNull(s1_2);
 			assertNotNull(s2_2);
 			
@@ -570,19 +570,19 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			assertEquals(1, testingSlotProvider.getNumberOfAvailableSlots());
 			
 			// schedule one more no-shared task
-			LogicalSlot sB0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 0, 3, null)), false, Collections.emptyList()).get();
+			LogicalSlot sB0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 0, 3, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			assertNotNull(sB0);
 			
 			// release the last of the original shared slots and allocate one more non-shared slot
 			s2_1.releaseSlot();
-			LogicalSlot sB2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 2, 3, null)), false, Collections.emptyList()).get();
+			LogicalSlot sB2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 2, 3, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			assertNotNull(sB2);
 			
 			
 			// release on non-shared and add some shared slots
 			sA2.releaseSlot();
-			LogicalSlot s1_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s2_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s1_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			assertNotNull(s1_3);
 			assertNotNull(s2_3);
 			
@@ -592,8 +592,8 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			s1_3.releaseSlot();
 			s2_3.releaseSlot();
 			
-			LogicalSlot sC0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 1, 2, null)), false, Collections.emptyList()).get();
-			LogicalSlot sC1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 0, 2, null)), false, Collections.emptyList()).get();
+			LogicalSlot sC0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 1, 2, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot sC1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 0, 2, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			assertNotNull(sC0);
 			assertNotNull(sC1);
 			
@@ -633,8 +633,8 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			TaskManagerLocation loc2 = testingSlotProvider.addTaskManager(2);
 			
 			// schedule one to each instance
-			LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1)).get();
-			LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2)).get();
+			LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get();
 			assertNotNull(s1);
 			assertNotNull(s2);
 			
@@ -643,8 +643,8 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			assertEquals(loc2, s2.getTaskManagerLocation());
 			
 			// schedule one from the other group to each instance
-			LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1)).get();
-			LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2)).get();
+			LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
+			LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get();
 			assertNotNull(s3);
 			assertNotNull(s4);
 			
@@ -679,8 +679,8 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			TaskManagerLocation loc2 = testingSlotProvider.addTaskManager(2);
 
 			// schedule one to each instance
-			LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1)).get();
-			LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1)).get();
+			LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 			assertNotNull(s1);
 			assertNotNull(s2);
 			
@@ -689,8 +689,8 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			assertEquals(loc1, s2.getTaskManagerLocation());
 			
 			// schedule one from the other group to each instance
-			LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2)).get();
-			LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2)).get();
+			LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get();
+			LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get();
 			assertNotNull(s3);
 			assertNotNull(s4);
 			
@@ -724,14 +724,14 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			TaskManagerLocation loc2 = testingSlotProvider.addTaskManager(2);
 
 			// schedule until the one instance is full
-			LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1)).get();
-			LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1)).get();
-			LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1)).get();
-			LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1)).get();
+			LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
+			LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
+			LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 
 			// schedule two more with preference of same instance --> need to go to other instance
-			LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 3, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1)).get();
-			LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 4, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1)).get();
+			LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 3, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
+			LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 4, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 			
 			assertNotNull(s1);
 			assertNotNull(s2);
@@ -775,19 +775,19 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			testingSlotProvider.addTaskManager(4);
 
 			// allocate something from group 1 and 2 interleaved with schedule for group 3
-			LogicalSlot slot_1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot slot_1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot slot_1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot slot_1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
-			LogicalSlot slot_2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot slot_2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot slot_2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot slot_2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
-			LogicalSlot slot_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot slot_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
-			LogicalSlot slot_1_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot slot_1_4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot slot_1_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot slot_1_4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
-			LogicalSlot slot_2_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot slot_2_4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot slot_2_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot slot_2_4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
 			// release groups 1 and 2
 			
@@ -803,10 +803,10 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			
 			// allocate group 4
 			
-			LogicalSlot slot_4_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot slot_4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot slot_4_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot slot_4_4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot slot_4_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot slot_4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot slot_4_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot slot_4_4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
 			// release groups 3 and 4
 			
@@ -855,7 +855,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 					@Override
 					public void run() {
 						try {
-							LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, enumerator4.getAndIncrement(), 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+							LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, enumerator4.getAndIncrement(), 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 							sleepUninterruptibly(rnd.nextInt(5));
 							slot.releaseSlot();
 
@@ -877,7 +877,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 					public void run() {
 						try {
 							if (flag3.compareAndSet(false, true)) {
-								LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+								LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 								sleepUninterruptibly(5);
 								
 								executor.execute(deploy4);
@@ -905,7 +905,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 					@Override
 					public void run() {
 						try {
-							LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, enumerator2.getAndIncrement(), 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+							LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, enumerator2.getAndIncrement(), 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 							// wait a bit till scheduling the successor
 							sleepUninterruptibly(rnd.nextInt(5));
@@ -932,7 +932,7 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 					@Override
 					public void run() {
 						try {
-							LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, enumerator1.getAndIncrement(), 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+							LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, enumerator1.getAndIncrement(), 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 							// wait a bit till scheduling the successor
 							sleepUninterruptibly(rnd.nextInt(5));
@@ -1005,27 +1005,27 @@ public class SchedulerSlotSharingTest extends SchedulerTestBase {
 			scheduler.newInstanceAvailable(getRandomInstance(4));
 			
 			// schedule one task for the first and second vertex
-			LogicalSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
 			assertEquals( s1.getTaskManagerLocation(), s2.getTaskManagerLocation() );
 			assertEquals(3, scheduler.getNumberOfAvailableSlots());
 			
-			LogicalSlot s3_0 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s3_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 1, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s4_0 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s4_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s3_0 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s3_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 1, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s4_0 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s4_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
 			s1.releaseSlot();
 			s2.releaseSlot();
 			
-			LogicalSlot s3_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 2, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s3_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 3, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s4_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-			LogicalSlot s4_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+			LogicalSlot s3_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 2, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s3_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 3, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s4_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+			LogicalSlot s4_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			
 			try {
-				scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+				scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 				fail("should throw an exception");
 			}
 			catch (ExecutionException e) {

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTest.java
index 8fd5f9e..a28e890 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTest.java
@@ -18,15 +18,30 @@
 
 package org.apache.flink.runtime.jobmanager.scheduler;
 
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.executiongraph.Execution;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+import org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils;
 import org.apache.flink.runtime.instance.Instance;
+import org.apache.flink.runtime.jobmaster.LogicalSlot;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.ExceptionUtils;
 import org.apache.flink.util.TestLogger;
 
+import org.hamcrest.Matchers;
 import org.junit.Test;
 
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+
 import static org.apache.flink.runtime.jobmanager.scheduler.SchedulerTestUtils.getRandomInstance;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.fail;
 
 public class SchedulerTest extends TestLogger {
@@ -88,4 +103,33 @@ public class SchedulerTest extends TestLogger {
 		assertFalse(i2.isAlive());
 		assertFalse(i3.isAlive());
 	}
+
+	/**
+	 * Tests that the Scheduler times out uncompleted slot futures.
+	 */
+	@Test
+	public void testSlotAllocationTimeout() throws Exception {
+		final Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutor());
+
+		final ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph();
+
+		final Map<ExecutionAttemptID, Execution> registeredExecutions = executionGraph.getRegisteredExecutions();
+
+		assertThat(registeredExecutions.values(), Matchers.not(Matchers.empty()));
+
+		final Execution execution = registeredExecutions.values().iterator().next();
+
+		final CompletableFuture<LogicalSlot> slotFuture = scheduler.allocateSlot(
+			new ScheduledUnit(
+				execution),
+			true,
+			Collections.emptyList(),
+			Time.milliseconds(1L));
+
+		try {
+			slotFuture.get();
+		} catch (ExecutionException ee) {
+			assertThat(ExceptionUtils.stripExecutionException(ee), Matchers.instanceOf(TimeoutException.class));
+		}
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTestBase.java
index d5460eb..949ff96 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTestBase.java
@@ -19,6 +19,7 @@
 package org.apache.flink.runtime.jobmanager.scheduler;
 
 import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
 import org.apache.flink.runtime.clusterframework.types.AllocationID;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
@@ -151,8 +152,8 @@ public class SchedulerTestBase extends TestLogger {
 		}
 
 		@Override
-		public CompletableFuture<LogicalSlot> allocateSlot(ScheduledUnit task, boolean allowQueued, Collection<TaskManagerLocation> preferredLocations) {
-			return scheduler.allocateSlot(task, allowQueued, preferredLocations);
+		public CompletableFuture<LogicalSlot> allocateSlot(ScheduledUnit task, boolean allowQueued, Collection<TaskManagerLocation> preferredLocations, Time allocationTimeout) {
+			return scheduler.allocateSlot(task, allowQueued, preferredLocations, allocationTimeout);
 		}
 
 		@Override
@@ -339,8 +340,8 @@ public class SchedulerTestBase extends TestLogger {
 		}
 
 		@Override
-		public CompletableFuture<LogicalSlot> allocateSlot(ScheduledUnit task, boolean allowQueued, Collection<TaskManagerLocation> preferredLocations) {
-			return slotProvider.allocateSlot(task, allowQueued, preferredLocations).thenApply(
+		public CompletableFuture<LogicalSlot> allocateSlot(ScheduledUnit task, boolean allowQueued, Collection<TaskManagerLocation> preferredLocations, Time allocationTimeout) {
+			return slotProvider.allocateSlot(task, allowQueued, preferredLocations, allocationTimeout).thenApply(
 				(LogicalSlot logicalSlot) -> {
 					switch (logicalSlot.getLocality()) {
 						case LOCAL:

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolCoLocationTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolCoLocationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolCoLocationTest.java
index 7454e3e..9a0256c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolCoLocationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolCoLocationTest.java
@@ -31,6 +31,7 @@ import org.apache.flink.runtime.resourcemanager.SlotRequest;
 import org.apache.flink.runtime.taskexecutor.slot.SlotOffer;
 import org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
 
 import org.junit.Test;
 
@@ -78,7 +79,8 @@ public class SlotPoolCoLocationTest extends SlotPoolSchedulingTestBase {
 				slotSharingGroupId,
 				coLocationConstraint1),
 			true,
-			Collections.emptyList());
+			Collections.emptyList(),
+			TestingUtils.infiniteTime());
 
 		CompletableFuture<LogicalSlot> logicalSlotFuture22 = slotProvider.allocateSlot(
 			new ScheduledUnit(
@@ -86,7 +88,8 @@ public class SlotPoolCoLocationTest extends SlotPoolSchedulingTestBase {
 				slotSharingGroupId,
 				coLocationConstraint2),
 			true,
-			Collections.emptyList());
+			Collections.emptyList(),
+			TestingUtils.infiniteTime());
 
 		CompletableFuture<LogicalSlot> logicalSlotFuture12 = slotProvider.allocateSlot(
 			new ScheduledUnit(
@@ -94,7 +97,8 @@ public class SlotPoolCoLocationTest extends SlotPoolSchedulingTestBase {
 				slotSharingGroupId,
 				coLocationConstraint1),
 			true,
-			Collections.emptyList());
+			Collections.emptyList(),
+			TestingUtils.infiniteTime());
 
 		CompletableFuture<LogicalSlot> logicalSlotFuture21 = slotProvider.allocateSlot(
 			new ScheduledUnit(
@@ -102,7 +106,8 @@ public class SlotPoolCoLocationTest extends SlotPoolSchedulingTestBase {
 				slotSharingGroupId,
 				coLocationConstraint2),
 			true,
-			Collections.emptyList());
+			Collections.emptyList(),
+			TestingUtils.infiniteTime());
 
 		final AllocationID allocationId1 = allocationIds.take();
 		final AllocationID allocationId2 = allocationIds.take();

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java
index d8e4487..9962250 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java
@@ -25,7 +25,6 @@ import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.clusterframework.types.AllocationID;
 import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway;
 import org.apache.flink.runtime.instance.SlotSharingGroupId;
-import org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException;
 import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit;
 import org.apache.flink.runtime.jobmanager.scheduler.SchedulerTestUtils;
 import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway;
@@ -62,6 +61,7 @@ import javax.annotation.Nullable;
 import java.util.Collections;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
 import java.util.function.Consumer;
 
 import static org.apache.flink.runtime.jobmaster.slotpool.AvailableSlotsTest.DEFAULT_TESTING_PROFILE;
@@ -111,7 +111,6 @@ public class SlotPoolRpcTest extends TestLogger {
 			jid,
 			SystemClock.getInstance(),
 			TestingUtils.infiniteTime(),
-			TestingUtils.infiniteTime(),
 			Time.milliseconds(10L), // this is the timeout for the request tested here
 			TestingUtils.infiniteTime()
 		);
@@ -131,7 +130,7 @@ public class SlotPoolRpcTest extends TestLogger {
 				future.get();
 				fail("We expected an ExecutionException.");
 			} catch (ExecutionException e) {
-				assertTrue(ExceptionUtils.stripExecutionException(e) instanceof NoResourceAvailableException);
+				assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException);
 			}
 		} finally {
 			RpcUtils.terminateRpcEndpoint(pool, timeout);
@@ -148,7 +147,6 @@ public class SlotPoolRpcTest extends TestLogger {
 			SystemClock.getInstance(),
 			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime(),
-			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime());
 
 		try {
@@ -191,7 +189,6 @@ public class SlotPoolRpcTest extends TestLogger {
 			SystemClock.getInstance(),
 			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime(),
-			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime());
 
 		try {
@@ -239,7 +236,6 @@ public class SlotPoolRpcTest extends TestLogger {
 			SystemClock.getInstance(),
 			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime(),
-			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime());
 
 		try {
@@ -307,7 +303,6 @@ public class SlotPoolRpcTest extends TestLogger {
 			rpcService,
 			jid,
 			SystemClock.getInstance(),
-			Time.milliseconds(10L),
 			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime());
@@ -328,7 +323,8 @@ public class SlotPoolRpcTest extends TestLogger {
 			CompletableFuture<LogicalSlot> future = pool.getSlotProvider().allocateSlot(
 				mockScheduledUnit,
 				true,
-				Collections.emptyList());
+				Collections.emptyList(),
+				Time.milliseconds(1L));
 
 			try {
 				future.get();
@@ -358,16 +354,14 @@ public class SlotPoolRpcTest extends TestLogger {
 				JobID jobId,
 				Clock clock,
 				Time slotRequestTimeout,
-				Time resourceManagerAllocationTimeout,
-				Time resourceManagerRequestTimeout,
+				Time rpcTimeout,
 				Time idleSlotTimeout) {
 			super(
 				rpcService,
 				jobId,
 				clock,
 				slotRequestTimeout,
-				resourceManagerAllocationTimeout,
-				resourceManagerRequestTimeout,
+				rpcTimeout,
 				idleSlotTimeout);
 
 			releaseSlotConsumer = null;

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolSlotSharingTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolSlotSharingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolSlotSharingTest.java
index bb6c2b7..b7fa484 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolSlotSharingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolSlotSharingTest.java
@@ -29,6 +29,7 @@ import org.apache.flink.runtime.resourcemanager.SlotRequest;
 import org.apache.flink.runtime.taskexecutor.slot.SlotOffer;
 import org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.util.ExceptionUtils;
 import org.apache.flink.util.FlinkException;
 
@@ -66,7 +67,8 @@ public class SlotPoolSlotSharingTest extends SlotPoolSchedulingTestBase {
 				slotSharingGroupId,
 				null),
 			true,
-			Collections.emptyList());
+			Collections.emptyList(),
+			TestingUtils.infiniteTime());
 
 		assertFalse(logicalSlotFuture.isDone());
 
@@ -102,7 +104,8 @@ public class SlotPoolSlotSharingTest extends SlotPoolSchedulingTestBase {
 				new SlotSharingGroupId(),
 				null),
 			true,
-			Collections.emptyList());
+			Collections.emptyList(),
+			TestingUtils.infiniteTime());
 
 		final AllocationID allocationId = allocationIdFuture.get();
 
@@ -140,7 +143,8 @@ public class SlotPoolSlotSharingTest extends SlotPoolSchedulingTestBase {
 				slotSharingGroupId,
 				null),
 			true,
-			Collections.emptyList());
+			Collections.emptyList(),
+			TestingUtils.infiniteTime());
 
 		CompletableFuture<LogicalSlot> logicalSlotFuture2 = slotProvider.allocateSlot(
 			new ScheduledUnit(
@@ -148,7 +152,8 @@ public class SlotPoolSlotSharingTest extends SlotPoolSchedulingTestBase {
 				slotSharingGroupId,
 				null),
 			true,
-			Collections.emptyList());
+			Collections.emptyList(),
+			TestingUtils.infiniteTime());
 
 		assertFalse(logicalSlotFuture1.isDone());
 		assertFalse(logicalSlotFuture2.isDone());
@@ -161,7 +166,8 @@ public class SlotPoolSlotSharingTest extends SlotPoolSchedulingTestBase {
 				slotSharingGroupId,
 				null),
 			true,
-			Collections.emptyList());
+			Collections.emptyList(),
+			TestingUtils.infiniteTime());
 
 		CompletableFuture<LogicalSlot> logicalSlotFuture4 = slotProvider.allocateSlot(
 			new ScheduledUnit(
@@ -169,7 +175,8 @@ public class SlotPoolSlotSharingTest extends SlotPoolSchedulingTestBase {
 				slotSharingGroupId,
 				null),
 			true,
-			Collections.emptyList());
+			Collections.emptyList(),
+			TestingUtils.infiniteTime());
 
 		assertFalse(logicalSlotFuture3.isDone());
 		assertFalse(logicalSlotFuture4.isDone());
@@ -235,7 +242,8 @@ public class SlotPoolSlotSharingTest extends SlotPoolSchedulingTestBase {
 				slotSharingGroupId1,
 				null),
 			true,
-			Collections.emptyList());
+			Collections.emptyList(),
+			TestingUtils.infiniteTime());
 
 		CompletableFuture<LogicalSlot> logicalSlotFuture2 = slotProvider.allocateSlot(
 			new ScheduledUnit(
@@ -243,7 +251,8 @@ public class SlotPoolSlotSharingTest extends SlotPoolSchedulingTestBase {
 				slotSharingGroupId1,
 				null),
 			true,
-			Collections.emptyList());
+			Collections.emptyList(),
+			TestingUtils.infiniteTime());
 
 		CompletableFuture<LogicalSlot> logicalSlotFuture3 = slotProvider.allocateSlot(
 			new ScheduledUnit(
@@ -251,7 +260,8 @@ public class SlotPoolSlotSharingTest extends SlotPoolSchedulingTestBase {
 				slotSharingGroupId2,
 				null),
 			true,
-			Collections.emptyList());
+			Collections.emptyList(),
+			TestingUtils.infiniteTime());
 
 		CompletableFuture<LogicalSlot> logicalSlotFuture4 = slotProvider.allocateSlot(
 			new ScheduledUnit(
@@ -259,7 +269,8 @@ public class SlotPoolSlotSharingTest extends SlotPoolSchedulingTestBase {
 				slotSharingGroupId2,
 				null),
 			true,
-			Collections.emptyList());
+			Collections.emptyList(),
+			TestingUtils.infiniteTime());
 
 		assertFalse(logicalSlotFuture1.isDone());
 		assertFalse(logicalSlotFuture2.isDone());

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java
index beb1de9..0c0b42c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java
@@ -66,6 +66,7 @@ import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import static org.apache.flink.runtime.jobmaster.slotpool.AvailableSlotsTest.DEFAULT_TESTING_PROFILE;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -588,7 +589,6 @@ public class SlotPoolTest extends TestLogger {
 			clock,
 			TestingUtils.infiniteTime(),
 			TestingUtils.infiniteTime(),
-			TestingUtils.infiniteTime(),
 			timeout);
 
 		try {
@@ -629,6 +629,37 @@ public class SlotPoolTest extends TestLogger {
 		}
 	}
 
+	/**
+	 * Tests that a slot allocation times out wrt to the specified time out.
+	 */
+	@Test
+	public void testSlotAllocationTimeout() throws Exception {
+		final SlotPool slotPool = new SlotPool(rpcService, jobId);
+
+		final Time allocationTimeout = Time.milliseconds(1L);
+
+		try {
+			setupSlotPool(slotPool, resourceManagerGateway);
+
+			final SlotProvider slotProvider = slotPool.getSlotProvider();
+
+			final CompletableFuture<LogicalSlot> allocationFuture = slotProvider.allocateSlot(
+				new DummyScheduledUnit(),
+				true,
+				Collections.emptyList(),
+				allocationTimeout);
+
+			try {
+				allocationFuture.get();
+				fail("Should have failed with a timeout exception.");
+			} catch (ExecutionException ee) {
+				assertThat(ExceptionUtils.stripExecutionException(ee), Matchers.instanceOf(TimeoutException.class));
+			}
+		} finally {
+			RpcUtils.terminateRpcEndpoint(slotPool, timeout);
+		}
+	}
+
 	private static SlotPoolGateway setupSlotPool(
 			SlotPool slotPool,
 			ResourceManagerGateway resourceManagerGateway) throws Exception {

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java
index 368dfee..01d88ef 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/partitioner/RescalePartitionerTest.java
@@ -155,7 +155,8 @@ public class RescalePartitionerTest extends TestLogger {
 			new RestartAllStrategy.Factory(),
 			new Scheduler(TestingUtils.defaultExecutionContext()),
 			ExecutionGraph.class.getClassLoader(),
-			VoidBlobWriter.getInstance());
+			VoidBlobWriter.getInstance(),
+			AkkaUtils.getDefaultTimeout());
 		try {
 			eg.attachJobGraph(jobVertices);
 		}


[22/22] flink git commit: [hotfix] [flip6] Remove unnecessary timeout from SlotPool

Posted by tr...@apache.org.
[hotfix] [flip6] Remove unnecessary timeout from SlotPool


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/341467dd
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/341467dd
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/341467dd

Branch: refs/heads/master
Commit: 341467dd58e4bdcfe133a31d8cb32b6a6a8d7559
Parents: 1321c17
Author: Till Rohrmann <tr...@apache.org>
Authored: Tue Feb 13 13:18:01 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Thu Feb 15 08:46:15 2018 +0100

----------------------------------------------------------------------
 .../apache/flink/configuration/JobManagerOptions.java | 14 ++------------
 .../org/apache/flink/runtime/jobmaster/JobMaster.java |  3 +--
 .../flink/runtime/jobmaster/slotpool/SlotPool.java    | 13 ++++++++-----
 .../runtime/jobmaster/slotpool/SlotPoolRpcTest.java   |  3 +--
 .../runtime/jobmaster/slotpool/SlotPoolTest.java      |  3 +--
 5 files changed, 13 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/341467dd/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
index a6df986..74ec5b7 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
@@ -135,22 +135,12 @@ public class JobManagerOptions {
 
 	public static final ConfigOption<Long> SLOT_REQUEST_TIMEOUT =
 		key("slot.request.timeout")
-		.defaultValue(10 * 60 * 1000L)
+		.defaultValue(5L * 60L * 1000L)
 		.withDescription("The timeout in milliseconds for requesting a slot from Slot Pool.");
 
-	public static final ConfigOption<Long> SLOT_REQUEST_RM_TIMEOUT =
-		key("slot.request.resourcemanager.timeout")
-			.defaultValue(10 * 1000L)
-			.withDescription("The timeout in milliseconds for sending a request to Resource Manager.");
-
-	public static final ConfigOption<Long> SLOT_ALLOCATION_RM_TIMEOUT =
-		key("slot.allocation.resourcemanager.timeout")
-			.defaultValue(5 * 60 * 1000L)
-			.withDescription("The timeout in milliseconds for allocation a slot from Resource Manager.");
-
 	public static final ConfigOption<Long> SLOT_IDLE_TIMEOUT =
 		key("slot.idle.timeout")
-			.defaultValue(5 * 60 * 1000L)
+			.defaultValue(20L * 1000L)
 			.withDescription("The timeout in milliseconds for a idle slot in Slot Pool.");
 
 	// ---------------------------------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/flink/blob/341467dd/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 6c5e524..8aa2617 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -280,8 +280,7 @@ public class JobMaster extends FencedRpcEndpoint<JobMasterId> implements JobMast
 			rpcService,
 			jobGraph.getJobID(),
 			SystemClock.getInstance(),
-			Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT)),
-			rpcTimeout,
+			rpcTimeout, Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT)),
 			Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_IDLE_TIMEOUT)));
 
 		this.slotPoolGateway = slotPool.getSelfGateway(SlotPoolGateway.class);

http://git-wip-us.apache.org/repos/asf/flink/blob/341467dd/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java
index 59f382f..72bb7e1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java
@@ -22,6 +22,7 @@ import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.configuration.JobManagerOptions;
+import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.clusterframework.types.AllocationID;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
@@ -137,8 +138,8 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 			rpcService,
 			jobId,
 			SystemClock.getInstance(),
-			Time.milliseconds(JobManagerOptions.SLOT_ALLOCATION_RM_TIMEOUT.defaultValue()),
-			Time.milliseconds(JobManagerOptions.SLOT_REQUEST_RM_TIMEOUT.defaultValue()),
+			AkkaUtils.getDefaultTimeout(),
+			Time.milliseconds(JobManagerOptions.SLOT_REQUEST_TIMEOUT.defaultValue()),
 			Time.milliseconds(JobManagerOptions.SLOT_IDLE_TIMEOUT.defaultValue()));
 	}
 
@@ -146,8 +147,8 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 			RpcService rpcService,
 			JobID jobId,
 			Clock clock,
-			Time slotRequestTimeout,
 			Time rpcTimeout,
+			Time slotRequestTimeout,
 			Time idleSlotTimeout) {
 
 		super(rpcService);
@@ -714,7 +715,9 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 	private void checkTimeoutSlotAllocation(SlotRequestId slotRequestID) {
 		PendingRequest request = pendingRequests.removeKeyA(slotRequestID);
 		if (request != null) {
-			failPendingRequest(request, new TimeoutException("Slot allocation request " + slotRequestID + " timed out"));
+			failPendingRequest(
+				request,
+				new TimeoutException("Slot allocation request " + slotRequestID + " timed out"));
 		}
 	}
 
@@ -733,7 +736,7 @@ public class SlotPool extends RpcEndpoint implements SlotPoolGateway, AllocatedS
 		if (request != null) {
 			failPendingRequest(
 				request,
-				new NoResourceAvailableException("No slot available and no connection to Resource Manager established."));
+				new TimeoutException("No slot available and no connection to Resource Manager established."));
 		}
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/341467dd/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java
index 9962250..bf8037d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java
@@ -360,8 +360,7 @@ public class SlotPoolRpcTest extends TestLogger {
 				rpcService,
 				jobId,
 				clock,
-				slotRequestTimeout,
-				rpcTimeout,
+				rpcTimeout, slotRequestTimeout,
 				idleSlotTimeout);
 
 			releaseSlotConsumer = null;

http://git-wip-us.apache.org/repos/asf/flink/blob/341467dd/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java
index 0c0b42c..6a85461 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java
@@ -587,8 +587,7 @@ public class SlotPoolTest extends TestLogger {
 			rpcService,
 			jobId,
 			clock,
-			TestingUtils.infiniteTime(),
-			TestingUtils.infiniteTime(),
+			TestingUtils.infiniteTime(), TestingUtils.infiniteTime(),
 			timeout);
 
 		try {


[10/22] flink git commit: [FLINK-8608] [flip6] Implement MiniDispatcher for job mode

Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
index f8f9164..c94e278 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
@@ -580,7 +580,7 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
 	}
 
 	private void checkInit() {
-		Preconditions.checkState(started, "The " + TaskSlotTable.class.getSimpleName() + " has to be started.");
+		Preconditions.checkState(started, "The %s has to be started.", TaskSlotTable.class.getSimpleName());
 	}
 
 	// ---------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
index 19efcbb..cee02fe 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
@@ -135,7 +135,7 @@ import java.util.concurrent.Executor;
  */
 public class WebMonitorEndpoint<T extends RestfulGateway> extends RestServerEndpoint implements LeaderContender {
 
-	protected final GatewayRetriever<T> leaderRetriever;
+	protected final GatewayRetriever<? extends T> leaderRetriever;
 	private final Configuration clusterConfiguration;
 	protected final RestHandlerConfiguration restConfiguration;
 	private final GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever;
@@ -145,7 +145,7 @@ public class WebMonitorEndpoint<T extends RestfulGateway> extends RestServerEndp
 	private final ExecutionGraphCache executionGraphCache;
 	private final CheckpointStatsCache checkpointStatsCache;
 
-	private final MetricFetcher<T> metricFetcher;
+	private final MetricFetcher<? extends T> metricFetcher;
 
 	private final LeaderElectionService leaderElectionService;
 
@@ -153,7 +153,7 @@ public class WebMonitorEndpoint<T extends RestfulGateway> extends RestServerEndp
 
 	public WebMonitorEndpoint(
 			RestServerEndpointConfiguration endpointConfiguration,
-			GatewayRetriever<T> leaderRetriever,
+			GatewayRetriever<? extends T> leaderRetriever,
 			Configuration clusterConfiguration,
 			RestHandlerConfiguration restConfiguration,
 			GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
index 8977906..a603340 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
@@ -186,7 +186,7 @@ public final class WebMonitorUtils {
 	 * @throws IOException if we cannot create the StaticFileServerHandler
 	 */
 	public static <T extends RestfulGateway> Optional<StaticFileServerHandler<T>> tryLoadWebContent(
-			GatewayRetriever<T> leaderRetriever,
+			GatewayRetriever<? extends T> leaderRetriever,
 			CompletableFuture<String> restAddressFuture,
 			Time timeout,
 			File tmpDir) throws IOException {

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java
index 8fe3c3b..6ea6383 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java
@@ -71,6 +71,8 @@ import org.junit.rules.TemporaryFolder;
 import org.junit.rules.TestName;
 import org.mockito.Mockito;
 
+import javax.annotation.Nullable;
+
 import java.util.Collection;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
@@ -345,8 +347,6 @@ public class DispatcherTest extends TestLogger {
 
 	private static class TestingDispatcher extends Dispatcher {
 
-		private final JobID expectedJobId;
-
 		private final CountDownLatch submitJobLatch = new CountDownLatch(2);
 
 		/**
@@ -372,19 +372,44 @@ public class DispatcherTest extends TestLogger {
 				endpointId,
 				configuration,
 				highAvailabilityServices,
+				highAvailabilityServices.getSubmittedJobGraphStore(),
 				resourceManagerGateway,
 				blobServer,
 				heartbeatServices,
 				metricRegistry,
 				archivedExecutionGraphStore,
+				new ExpectedJobIdJobManagerRunnerFactory(expectedJobId),
 				fatalErrorHandler,
 				null);
+		}
+
+		@Override
+		public CompletableFuture<Acknowledge> submitJob(final JobGraph jobGraph, final Time timeout) {
+			final CompletableFuture<Acknowledge> submitJobFuture = super.submitJob(jobGraph, timeout);
+
+			submitJobFuture.thenAccept(ignored -> submitJobLatch.countDown());
+
+			return submitJobFuture;
+		}
+
+		@Override
+		void recoverJobs() {
+			if (recoverJobsEnabled.get()) {
+				super.recoverJobs();
+			}
+		}
+	}
 
+	private static final class ExpectedJobIdJobManagerRunnerFactory implements Dispatcher.JobManagerRunnerFactory {
+
+		private final JobID expectedJobId;
+
+		private ExpectedJobIdJobManagerRunnerFactory(JobID expectedJobId) {
 			this.expectedJobId = expectedJobId;
 		}
 
 		@Override
-		protected JobManagerRunner createJobManagerRunner(
+		public JobManagerRunner createJobManagerRunner(
 				ResourceID resourceId,
 				JobGraph jobGraph,
 				Configuration configuration,
@@ -395,10 +420,11 @@ public class DispatcherTest extends TestLogger {
 				JobManagerSharedServices jobManagerSharedServices,
 				MetricRegistry metricRegistry,
 				OnCompletionActions onCompleteActions,
-				FatalErrorHandler fatalErrorHandler) throws Exception {
+				FatalErrorHandler fatalErrorHandler,
+				@Nullable String restAddress) throws Exception {
 			assertEquals(expectedJobId, jobGraph.getJobID());
 
-			return new JobManagerRunner(
+			return Dispatcher.DefaultJobManagerRunnerFactory.INSTANCE.createJobManagerRunner(
 				resourceId,
 				jobGraph,
 				configuration,
@@ -410,23 +436,7 @@ public class DispatcherTest extends TestLogger {
 				metricRegistry,
 				onCompleteActions,
 				fatalErrorHandler,
-				null);
-		}
-
-		@Override
-		public CompletableFuture<Acknowledge> submitJob(final JobGraph jobGraph, final Time timeout) {
-			final CompletableFuture<Acknowledge> submitJobFuture = super.submitJob(jobGraph, timeout);
-
-			submitJobFuture.thenAccept(ignored -> submitJobLatch.countDown());
-
-			return submitJobFuture;
-		}
-
-		@Override
-		void recoverJobs() {
-			if (recoverJobsEnabled.get()) {
-				super.recoverJobs();
-			}
+				restAddress);
 		}
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MemoryArchivedExecutionGraphStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MemoryArchivedExecutionGraphStore.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MemoryArchivedExecutionGraphStore.java
deleted file mode 100644
index bcb7df2..0000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MemoryArchivedExecutionGraphStore.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.dispatcher;
-
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph;
-import org.apache.flink.runtime.jobgraph.JobStatus;
-import org.apache.flink.runtime.messages.webmonitor.JobDetails;
-import org.apache.flink.runtime.messages.webmonitor.JobsOverview;
-import org.apache.flink.runtime.webmonitor.WebMonitorUtils;
-
-import javax.annotation.Nullable;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/**
- * {@link ArchivedExecutionGraphStore} implementation which stores the {@link ArchivedExecutionGraph}
- * in memory.
- */
-public class MemoryArchivedExecutionGraphStore implements ArchivedExecutionGraphStore {
-
-	private final Map<JobID, ArchivedExecutionGraph> serializableExecutionGraphs = new HashMap<>(4);
-
-	@Override
-	public int size() {
-		return serializableExecutionGraphs.size();
-	}
-
-	@Nullable
-	@Override
-	public ArchivedExecutionGraph get(JobID jobId) {
-		return serializableExecutionGraphs.get(jobId);
-	}
-
-	@Override
-	public void put(ArchivedExecutionGraph serializableExecutionGraph) throws IOException {
-		serializableExecutionGraphs.put(serializableExecutionGraph.getJobID(), serializableExecutionGraph);
-	}
-
-	@Override
-	public JobsOverview getStoredJobsOverview() {
-		Collection<JobStatus> allJobStatus = serializableExecutionGraphs.values().stream()
-			.map(ArchivedExecutionGraph::getState)
-			.collect(Collectors.toList());
-
-		return JobsOverview.create(allJobStatus);
-	}
-
-	@Override
-	public Collection<JobDetails> getAvailableJobDetails() {
-		return serializableExecutionGraphs.values().stream()
-			.map(WebMonitorUtils::createDetailsForJob)
-			.collect(Collectors.toList());
-	}
-
-	@Nullable
-	@Override
-	public JobDetails getAvailableJobDetails(JobID jobId) {
-		final ArchivedExecutionGraph archivedExecutionGraph = serializableExecutionGraphs.get(jobId);
-
-		if (archivedExecutionGraph != null) {
-			return WebMonitorUtils.createDetailsForJob(archivedExecutionGraph);
-		} else {
-			return null;
-		}
-	}
-
-	@Override
-	public void close() throws IOException {
-		serializableExecutionGraphs.clear();
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java
new file mode 100644
index 0000000..bc2478d
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.dispatcher;
+
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.BlobServerOptions;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.blob.BlobServer;
+import org.apache.flink.runtime.blob.VoidBlobStore;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.entrypoint.ClusterEntrypoint;
+import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph;
+import org.apache.flink.runtime.heartbeat.HeartbeatServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.jobmanager.OnCompletionActions;
+import org.apache.flink.runtime.jobmaster.JobManagerRunner;
+import org.apache.flink.runtime.jobmaster.JobManagerSharedServices;
+import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.metrics.NoOpMetricRegistry;
+import org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway;
+import org.apache.flink.runtime.rest.handler.legacy.utils.ArchivedExecutionGraphBuilder;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
+import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.runtime.rpc.RpcUtils;
+import org.apache.flink.runtime.rpc.TestingRpcService;
+import org.apache.flink.runtime.util.TestingFatalErrorHandler;
+import org.apache.flink.testutils.category.Flip6;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TemporaryFolder;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Tests for the {@link MiniDispatcher}.
+ */
+@Category(Flip6.class)
+public class MiniDispatcherTest extends TestLogger {
+
+	private static final JobGraph jobGraph = new JobGraph();
+
+	private static final Time timeout = Time.seconds(10L);
+
+	@ClassRule
+	public static TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+	private static TestingRpcService rpcService;
+
+	private static Configuration configuration;
+
+	private static BlobServer blobServer;
+
+	private MiniDispatcher miniDispatcher;
+
+	private CompletableFuture<JobGraph> jobGraphFuture;
+
+	private TestingLeaderElectionService dispatcherLeaderElectionService;
+
+	@BeforeClass
+	public static void setupClass() throws IOException {
+		rpcService = new TestingRpcService();
+		configuration = new Configuration();
+
+		configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
+
+		blobServer = new BlobServer(configuration, new VoidBlobStore());
+	}
+
+	@Before
+	public void setup() throws Exception {
+		dispatcherLeaderElectionService = new TestingLeaderElectionService();
+		final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
+		final TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
+		final HeartbeatServices heartbeatServices = new HeartbeatServices(1000L, 1000L);
+		final ArchivedExecutionGraphStore archivedExecutionGraphStore = new MemoryArchivedExecutionGraphStore();
+		final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
+
+		highAvailabilityServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);
+
+		jobGraphFuture = new CompletableFuture<>();
+		final TestingJobManagerRunnerFactory testingJobManagerRunnerFactory = new TestingJobManagerRunnerFactory(jobGraphFuture);
+
+		miniDispatcher = new MiniDispatcher(
+			rpcService,
+			UUID.randomUUID().toString(),
+			configuration,
+			highAvailabilityServices,
+			resourceManagerGateway,
+			blobServer,
+			heartbeatServices,
+			NoOpMetricRegistry.INSTANCE,
+			archivedExecutionGraphStore,
+			testingJobManagerRunnerFactory,
+			testingFatalErrorHandler,
+			null,
+			jobGraph,
+			ClusterEntrypoint.ExecutionMode.DETACHED);
+
+		miniDispatcher.start();
+	}
+
+	@After
+	public void teardown() throws InterruptedException, ExecutionException, TimeoutException {
+		if (miniDispatcher != null) {
+			RpcUtils.terminateRpcEndpoint(miniDispatcher, timeout);
+			miniDispatcher = null;
+		}
+	}
+
+	@AfterClass
+	public static void teardownClass() throws IOException {
+		blobServer.close();
+		rpcService.stopService();
+	}
+
+	/**
+	 * Tests that the {@link MiniDispatcher} recovers the single job with which it
+	 * was started.
+	 */
+	@Test
+	public void testSingleJobRecovery() throws Exception {
+		// wait until the Dispatcher is the leader
+		dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get();
+
+		final JobGraph actualJobGraph = jobGraphFuture.get();
+
+		assertThat(actualJobGraph.getJobID(), is(jobGraph.getJobID()));
+	}
+
+	/**
+	 * Tests that in detached mode, the {@link MiniDispatcher} will terminate after the job
+	 * has completed.
+	 */
+	@Test
+	public void testTerminationAfterJobCompletion() throws Exception {
+		final Dispatcher.DispatcherOnCompleteActions completeActions = miniDispatcher.new DispatcherOnCompleteActions(jobGraph.getJobID());
+
+		final ArchivedExecutionGraph archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
+			.setJobID(jobGraph.getJobID())
+			.setState(JobStatus.FINISHED)
+			.build();
+
+		// wait until the Dispatcher is the leader
+		dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get();
+
+		// wait until we have submitted the job
+		jobGraphFuture.get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);
+
+		completeActions.jobReachedGloballyTerminalState(archivedExecutionGraph);
+
+		final CompletableFuture<Boolean> terminationFuture = miniDispatcher.getTerminationFuture();
+
+		// wait until we terminate
+		terminationFuture.get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);
+	}
+
+	private static final class TestingJobManagerRunnerFactory implements Dispatcher.JobManagerRunnerFactory {
+
+		private final CompletableFuture<JobGraph> jobGraphFuture;
+
+		private TestingJobManagerRunnerFactory(CompletableFuture<JobGraph> jobGraphFuture) {
+			this.jobGraphFuture = jobGraphFuture;
+		}
+
+		@Override
+		public JobManagerRunner createJobManagerRunner(ResourceID resourceId, JobGraph jobGraph, Configuration configuration, RpcService rpcService, HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, BlobServer blobServer, JobManagerSharedServices jobManagerSharedServices, MetricRegistry metricRegistry, OnCompletionActions onCompleteActions, FatalErrorHandler fatalErrorHandler, @Nullable String restAddress) throws Exception {
+			jobGraphFuture.complete(jobGraph);
+
+			return mock(JobManagerRunner.class);
+		}
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnClusterDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnClusterDescriptor.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnClusterDescriptor.java
index 95a31be..ec41d8e 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnClusterDescriptor.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnClusterDescriptor.java
@@ -74,7 +74,10 @@ public class TestingYarnClusterDescriptor extends YarnClusterDescriptor {
 	}
 
 	@Override
-	public YarnClusterClient deployJobCluster(ClusterSpecification clusterSpecification, JobGraph jobGraph) {
+	public YarnClusterClient deployJobCluster(
+			ClusterSpecification clusterSpecification,
+			JobGraph jobGraph,
+			boolean detached) {
 		throw new UnsupportedOperationException("Cannot deploy a per-job cluster yet.");
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
index bea9001..037e086 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
@@ -85,7 +85,10 @@ public class YARNITCase extends YarnTestBase {
 
 			jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI()));
 
-			ClusterClient<ApplicationId> clusterClient = flip6YarnClusterDescriptor.deployJobCluster(clusterSpecification, jobGraph);
+			ClusterClient<ApplicationId> clusterClient = flip6YarnClusterDescriptor.deployJobCluster(
+				clusterSpecification,
+				jobGraph,
+				true);
 
 			clusterClient.shutdown();
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
index f015235..583ee54 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java
@@ -35,6 +35,7 @@ import org.apache.flink.configuration.RestOptions;
 import org.apache.flink.configuration.SecurityOptions;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.clusterframework.BootstrapTools;
+import org.apache.flink.runtime.entrypoint.ClusterEntrypoint;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
 import org.apache.flink.util.FlinkException;
@@ -384,25 +385,14 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
 			return deployInternal(
 				clusterSpecification,
 				getYarnSessionClusterEntrypoint(),
-				null);
+				null,
+				false);
 		} catch (Exception e) {
 			throw new ClusterDeploymentException("Couldn't deploy Yarn session cluster", e);
 		}
 	}
 
 	@Override
-	public ClusterClient<ApplicationId> deployJobCluster(ClusterSpecification clusterSpecification, JobGraph jobGraph) throws ClusterDeploymentException {
-		try {
-			return deployInternal(
-				clusterSpecification,
-				getYarnJobClusterEntrypoint(),
-				jobGraph);
-		} catch (Exception e) {
-			throw new ClusterDeploymentException("Could not deploy Yarn job cluster.", e);
-		}
-	}
-
-	@Override
 	public void terminateCluster(ApplicationId applicationId) throws FlinkException {
 		try {
 			yarnClient.killApplication(applicationId);
@@ -416,12 +406,15 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
 	 * deployed on YARN.
 	 *
 	 * @param clusterSpecification Initial cluster specification for the to be deployed Flink cluster
-	 * @param jobGraph A job graph which is deployed with the Flink cluster, null if none
+	 * @param yarnClusterEntrypoint Class name of the Yarn cluster entry point.
+	 * @param jobGraph A job graph which is deployed with the Flink cluster, {@code null} if none
+	 * @param detached True if the cluster should be started in detached mode
 	 */
 	protected ClusterClient<ApplicationId> deployInternal(
 			ClusterSpecification clusterSpecification,
 			String yarnClusterEntrypoint,
-			@Nullable JobGraph jobGraph) throws Exception {
+			@Nullable JobGraph jobGraph,
+			boolean detached) throws Exception {
 
 		if (UserGroupInformation.isSecurityEnabled()) {
 			// note: UGI::hasKerberosCredentials inaccurately reports false
@@ -482,7 +475,14 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
 
 		LOG.info("Cluster specification: {}", validClusterSpecification);
 
+		final ClusterEntrypoint.ExecutionMode executionMode = detached ?
+			ClusterEntrypoint.ExecutionMode.DETACHED
+			: ClusterEntrypoint.ExecutionMode.NORMAL;
+
+		flinkConfiguration.setString(ClusterEntrypoint.EXECUTION_MODE, executionMode.toString());
+
 		ApplicationReport report = startAppMaster(
+			flinkConfiguration,
 			yarnClusterEntrypoint,
 			jobGraph,
 			yarnClient,
@@ -635,6 +635,7 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
 	}
 
 	public ApplicationReport startAppMaster(
+			Configuration configuration,
 			String yarnClusterEntrypoint,
 			JobGraph jobGraph,
 			YarnClient yarnClient,
@@ -644,7 +645,7 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
 		// ------------------ Initialize the file systems -------------------------
 
 		try {
-			org.apache.flink.core.fs.FileSystem.initialize(flinkConfiguration);
+			org.apache.flink.core.fs.FileSystem.initialize(configuration);
 		} catch (IOException e) {
 			throw new IOException("Error while setting the default " +
 					"filesystem scheme from configuration.", e);
@@ -699,16 +700,16 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
 		// no user specified cli argument for namespace?
 		if (zkNamespace == null || zkNamespace.isEmpty()) {
 			// namespace defined in config? else use applicationId as default.
-			zkNamespace = flinkConfiguration.getString(HighAvailabilityOptions.HA_CLUSTER_ID, String.valueOf(appId));
+			zkNamespace = configuration.getString(HighAvailabilityOptions.HA_CLUSTER_ID, String.valueOf(appId));
 			setZookeeperNamespace(zkNamespace);
 		}
 
-		flinkConfiguration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zkNamespace);
+		configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zkNamespace);
 
-		if (HighAvailabilityMode.isHighAvailabilityModeActivated(flinkConfiguration)) {
+		if (HighAvailabilityMode.isHighAvailabilityModeActivated(configuration)) {
 			// activate re-execution of failed applications
 			appContext.setMaxAppAttempts(
-				flinkConfiguration.getInteger(
+				configuration.getInteger(
 					YarnConfigOptions.APPLICATION_ATTEMPTS.key(),
 					YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS));
 
@@ -716,7 +717,7 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
 		} else {
 			// set number of application retries to 1 in the default case
 			appContext.setMaxAppAttempts(
-				flinkConfiguration.getInteger(
+				configuration.getInteger(
 					YarnConfigOptions.APPLICATION_ATTEMPTS.key(),
 					1));
 		}
@@ -797,7 +798,7 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
 		// write out configuration file
 		File tmpConfigurationFile = File.createTempFile(appId + "-flink-conf.yaml", null);
 		tmpConfigurationFile.deleteOnExit();
-		BootstrapTools.writeConfiguration(flinkConfiguration, tmpConfigurationFile);
+		BootstrapTools.writeConfiguration(configuration, tmpConfigurationFile);
 
 		Path remotePathConf = setupSingleLocalResource(
 			"flink-conf.yaml",
@@ -884,7 +885,7 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
 
 		// setup security tokens
 		Path remotePathKeytab = null;
-		String keytab = flinkConfiguration.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB);
+		String keytab = configuration.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB);
 		if (keytab != null) {
 			LOG.info("Adding keytab {} to the AM container local resource bucket", keytab);
 			remotePathKeytab = setupSingleLocalResource(
@@ -916,7 +917,7 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
 		// Setup CLASSPATH and environment variables for ApplicationMaster
 		final Map<String, String> appMasterEnv = new HashMap<>();
 		// set user specified app master environment variables
-		appMasterEnv.putAll(Utils.getEnvironmentVariables(ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX, flinkConfiguration));
+		appMasterEnv.putAll(Utils.getEnvironmentVariables(ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX, configuration));
 		// set Flink app class path
 		appMasterEnv.put(YarnConfigKeys.ENV_FLINK_CLASSPATH, classPathBuilder.toString());
 
@@ -937,7 +938,7 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor
 
 		if (remotePathKeytab != null) {
 			appMasterEnv.put(YarnConfigKeys.KEYTAB_PATH, remotePathKeytab.toString());
-			String principal = flinkConfiguration.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL);
+			String principal = configuration.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL);
 			appMasterEnv.put(YarnConfigKeys.KEYTAB_PRINCIPAL, principal);
 		}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-yarn/src/main/java/org/apache/flink/yarn/Flip6YarnClusterDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/Flip6YarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/Flip6YarnClusterDescriptor.java
index e3f9646..461dd55 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/Flip6YarnClusterDescriptor.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/Flip6YarnClusterDescriptor.java
@@ -18,9 +18,12 @@
 
 package org.apache.flink.yarn;
 
+import org.apache.flink.client.deployment.ClusterDeploymentException;
+import org.apache.flink.client.deployment.ClusterSpecification;
 import org.apache.flink.client.program.ClusterClient;
 import org.apache.flink.client.program.rest.RestClusterClient;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.yarn.entrypoint.YarnJobClusterEntrypoint;
 import org.apache.flink.yarn.entrypoint.YarnSessionClusterEntrypoint;
 
@@ -52,6 +55,27 @@ public class Flip6YarnClusterDescriptor extends AbstractYarnClusterDescriptor {
 	}
 
 	@Override
+	public ClusterClient<ApplicationId> deployJobCluster(
+		ClusterSpecification clusterSpecification,
+		JobGraph jobGraph,
+		boolean detached) throws ClusterDeploymentException {
+
+		// this is required to work with Flip-6 because the slots are allocated
+		// lazily
+		jobGraph.setAllowQueuedScheduling(true);
+
+		try {
+			return deployInternal(
+				clusterSpecification,
+				getYarnJobClusterEntrypoint(),
+				jobGraph,
+				detached);
+		} catch (Exception e) {
+			throw new ClusterDeploymentException("Could not deploy Yarn job cluster.", e);
+		}
+	}
+
+	@Override
 	protected ClusterClient<ApplicationId> createYarnClusterClient(
 			AbstractYarnClusterDescriptor descriptor,
 			int numberTaskManagers,

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
index 9467326..a5254a0 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
@@ -50,7 +50,10 @@ public class YarnClusterDescriptor extends AbstractYarnClusterDescriptor {
 	}
 
 	@Override
-	public YarnClusterClient deployJobCluster(ClusterSpecification clusterSpecification, JobGraph jobGraph) {
+	public YarnClusterClient deployJobCluster(
+			ClusterSpecification clusterSpecification,
+			JobGraph jobGraph,
+			boolean detached) {
 		throw new UnsupportedOperationException("Cannot deploy a per-job yarn cluster yet.");
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/84418d0b/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
index 0859f03..8db3740 100644
--- a/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
@@ -18,9 +18,12 @@
 
 package org.apache.flink.yarn;
 
+import org.apache.flink.client.deployment.ClusterDeploymentException;
 import org.apache.flink.client.deployment.ClusterRetrieveException;
+import org.apache.flink.client.deployment.ClusterSpecification;
 import org.apache.flink.client.program.ClusterClient;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.util.Preconditions;
 import org.apache.flink.util.TestLogger;
 
@@ -137,5 +140,10 @@ public class AbstractYarnClusterTest extends TestLogger {
 		protected ClusterClient<ApplicationId> createYarnClusterClient(AbstractYarnClusterDescriptor descriptor, int numberTaskManagers, int slotsPerTaskManager, ApplicationReport report, Configuration flinkConfiguration, boolean perJobCluster) throws Exception {
 			throw new UnsupportedOperationException("Not needed for testing");
 		}
+
+		@Override
+		public ClusterClient<ApplicationId> deployJobCluster(ClusterSpecification clusterSpecification, JobGraph jobGraph, boolean detached) throws ClusterDeploymentException {
+			throw new UnsupportedOperationException("Not needed for testing");
+		}
 	}
 }


[15/22] flink git commit: [FLINK-7713][flip6] Implement JarUploadHandler

Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandlerTest.java
deleted file mode 100644
index a067d8f..0000000
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandlerTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.webmonitor.handlers;
-
-import org.apache.flink.runtime.concurrent.Executors;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Tests for the JarDeleteHandler.
- */
-public class JarDeleteHandlerTest {
-	@Test
-	public void testGetPaths() {
-		JarDeleteHandler handler = new JarDeleteHandler(Executors.directExecutor(), null);
-		String[] paths = handler.getPaths();
-		Assert.assertEquals(1, paths.length);
-		Assert.assertEquals("/jars/:jarid", paths[0]);
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandlerTest.java
deleted file mode 100644
index 5da4913..0000000
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarListHandlerTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.webmonitor.handlers;
-
-import org.apache.flink.runtime.concurrent.Executors;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Tests for the JarListHandler.
- */
-public class JarListHandlerTest {
-	@Test
-	public void testGetPaths() {
-		JarListHandler handler = new JarListHandler(Executors.directExecutor(), null);
-		String[] paths = handler.getPaths();
-		Assert.assertEquals(1, paths.length);
-		Assert.assertEquals("/jars", paths[0]);
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarPlanHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarPlanHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarPlanHandlerTest.java
deleted file mode 100644
index f5ed339..0000000
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarPlanHandlerTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.webmonitor.handlers;
-
-import org.apache.flink.runtime.concurrent.Executors;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Tests for the JarPlanHandler.
- */
-public class JarPlanHandlerTest {
-	@Test
-	public void testGetPaths() {
-		JarPlanHandler handler = new JarPlanHandler(Executors.directExecutor(), null);
-		String[] paths = handler.getPaths();
-		Assert.assertEquals(1, paths.length);
-		Assert.assertEquals("/jars/:jarid/plan", paths[0]);
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java
deleted file mode 100644
index 67dad13..0000000
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.webmonitor.handlers;
-
-import org.apache.flink.api.common.time.Time;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.concurrent.Executors;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Tests for the JarRunHandler.
- */
-public class JarRunHandlerTest {
-	@Test
-	public void testGetPaths() {
-		JarRunHandler handler = new JarRunHandler(Executors.directExecutor(), null, Time.seconds(0L), new Configuration());
-		String[] paths = handler.getPaths();
-		Assert.assertEquals(1, paths.length);
-		Assert.assertEquals("/jars/:jarid/run", paths[0]);
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadHandlerTest.java
index ea8b524..fd8224c 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadHandlerTest.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadHandlerTest.java
@@ -18,20 +18,117 @@
 
 package org.apache.flink.runtime.webmonitor.handlers;
 
+import org.apache.flink.api.common.time.Time;
 import org.apache.flink.runtime.concurrent.Executors;
+import org.apache.flink.runtime.dispatcher.DispatcherGateway;
+import org.apache.flink.runtime.rest.handler.HandlerRequest;
+import org.apache.flink.runtime.rest.handler.HandlerRequestException;
+import org.apache.flink.runtime.rest.handler.RestHandlerException;
+import org.apache.flink.runtime.rest.messages.EmptyMessageParameters;
+import org.apache.flink.runtime.rest.messages.FileUpload;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.TestLogger;
 
-import org.junit.Assert;
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus;
+
+import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
 
 /**
- * Tests for the JarUploadHandler.
+ * Tests for {@link JarUploadHandler}.
  */
-public class JarUploadHandlerTest {
+public class JarUploadHandlerTest extends TestLogger {
+
+	@Rule
+	public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+	private JarUploadHandler jarUploadHandler;
+
+	@Mock
+	private DispatcherGateway mockDispatcherGateway;
+
+	private Path jarDir;
+
+	@Before
+	public void setUp() throws Exception {
+		MockitoAnnotations.initMocks(this);
+
+		jarDir = temporaryFolder.newFolder().toPath();
+		jarUploadHandler = new JarUploadHandler(
+			CompletableFuture.completedFuture("localhost:12345"),
+			() -> CompletableFuture.completedFuture(mockDispatcherGateway),
+			Time.seconds(10),
+			Collections.emptyMap(),
+			new JarUploadMessageHeaders(),
+			jarDir,
+			Executors.directExecutor());
+	}
+
 	@Test
-	public void testGetPaths() {
-		JarUploadHandler handler = new JarUploadHandler(Executors.directExecutor(), null);
-		String[] paths = handler.getPaths();
-		Assert.assertEquals(1, paths.length);
-		Assert.assertEquals("/jars/upload", paths[0]);
+	public void testRejectNonJarFiles() throws Exception {
+		final Path uploadedFile = Files.createFile(jarDir.resolve("katrin.png"));
+		final HandlerRequest<FileUpload, EmptyMessageParameters> request = createRequest(uploadedFile);
+
+		try {
+			jarUploadHandler.handleRequest(request, mockDispatcherGateway).get();
+			fail("Expected exception not thrown.");
+		} catch (final ExecutionException e) {
+			final Throwable throwable = ExceptionUtils.stripCompletionException(e.getCause());
+			assertThat(throwable, instanceOf(RestHandlerException.class));
+			final RestHandlerException restHandlerException = (RestHandlerException) throwable;
+			assertThat(restHandlerException.getHttpResponseStatus(), equalTo(HttpResponseStatus.BAD_REQUEST));
+		}
+	}
+
+	@Test
+	public void testUploadJar() throws Exception {
+		final Path uploadedFile = Files.createFile(jarDir.resolve("Kafka010Example.jar"));
+		final HandlerRequest<FileUpload, EmptyMessageParameters> request = createRequest(uploadedFile);
+
+		final JarUploadResponseBody jarUploadResponseBody = jarUploadHandler.handleRequest(request, mockDispatcherGateway).get();
+		assertThat(jarUploadResponseBody.getStatus(), equalTo(JarUploadResponseBody.UploadStatus.success));
+		assertThat(jarUploadResponseBody.getFilename(), equalTo(uploadedFile.normalize().toString()));
+	}
+
+	@Test
+	public void testFailedUpload() throws Exception {
+		final Path uploadedFile = jarDir.resolve("Kafka010Example.jar");
+		final HandlerRequest<FileUpload, EmptyMessageParameters> request = createRequest(uploadedFile);
+
+		try {
+			jarUploadHandler.handleRequest(request, mockDispatcherGateway).get();
+			fail("Expected exception not thrown.");
+		} catch (final ExecutionException e) {
+			final Throwable throwable = ExceptionUtils.stripCompletionException(e.getCause());
+			assertThat(throwable, instanceOf(RestHandlerException.class));
+			final RestHandlerException restHandlerException = (RestHandlerException) throwable;
+			assertThat(restHandlerException.getMessage(), containsString("Could not move uploaded jar file"));
+			assertThat(restHandlerException.getHttpResponseStatus(), equalTo(HttpResponseStatus.INTERNAL_SERVER_ERROR));
+		}
+	}
+
+	private static HandlerRequest<FileUpload, EmptyMessageParameters> createRequest(
+			final Path uploadedFile) throws HandlerRequestException {
+		return new HandlerRequest<>(
+			new FileUpload(uploadedFile),
+			EmptyMessageParameters.getInstance(),
+			Collections.emptyMap(),
+			Collections.emptyMap());
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadResponseBodyTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadResponseBodyTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadResponseBodyTest.java
new file mode 100644
index 0000000..cfdc140
--- /dev/null
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarUploadResponseBodyTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers;
+
+import org.apache.flink.runtime.rest.messages.RestResponseMarshallingTestBase;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for {@link JarUploadResponseBody}.
+ */
+public class JarUploadResponseBodyTest extends RestResponseMarshallingTestBase<JarUploadResponseBody> {
+
+	@Override
+	protected Class<JarUploadResponseBody> getTestResponseClass() {
+		return JarUploadResponseBody.class;
+	}
+
+	@Override
+	protected JarUploadResponseBody getTestResponseInstance() throws Exception {
+		return new JarUploadResponseBody("/tmp");
+	}
+
+	@Override
+	protected void assertOriginalEqualsToUnmarshalled(final JarUploadResponseBody expected, final JarUploadResponseBody actual) {
+		assertEquals(expected.getFilename(), actual.getFilename());
+		assertEquals(expected.getStatus(), actual.getStatus());
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarAccessDeniedHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarAccessDeniedHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarAccessDeniedHandlerTest.java
new file mode 100644
index 0000000..86874d3
--- /dev/null
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarAccessDeniedHandlerTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers.legacy;
+
+import org.apache.flink.runtime.concurrent.Executors;
+
+import org.apache.flink.shaded.guava18.com.google.common.collect.Lists;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.List;
+
+/**
+ * Tests for the JarAccessDeniedHandler.
+ */
+public class JarAccessDeniedHandlerTest {
+	@Test
+	public void testGetPaths() {
+		JarAccessDeniedHandler handler = new JarAccessDeniedHandler(Executors.directExecutor());
+		String[] paths = handler.getPaths();
+		Assert.assertEquals(5, paths.length);
+		List<String> pathsList = Lists.newArrayList(paths);
+		Assert.assertTrue(pathsList.contains("/jars"));
+		Assert.assertTrue(pathsList.contains("/jars/upload"));
+		Assert.assertTrue(pathsList.contains("/jars/:jarid"));
+		Assert.assertTrue(pathsList.contains("/jars/:jarid/plan"));
+		Assert.assertTrue(pathsList.contains("/jars/:jarid/run"));
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarActionHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarActionHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarActionHandlerTest.java
new file mode 100644
index 0000000..63a91a3
--- /dev/null
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarActionHandlerTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers.legacy;
+
+import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for the JarActionHandler.
+ */
+public class JarActionHandlerTest {
+
+	/**
+	 * Test that the savepoint settings are correctly parsed.
+	 */
+	@Test
+	public void testSavepointRestoreSettings() throws Exception {
+		Map<String, String> pathParams = new HashMap<>();
+		pathParams.put("jarid", "required"); // required
+
+		// the following should be ignored, because they are parsed from the query params
+		pathParams.put("savepointPath", "ignored");
+		pathParams.put("allowNonRestoredState", "ignored");
+
+		Map<String, String> queryParams = new HashMap<>(); // <-- everything goes here
+
+		// Nothing configured
+		JarActionHandler.JarActionHandlerConfig config = JarActionHandler.JarActionHandlerConfig.fromParams(pathParams, queryParams);
+		assertEquals(SavepointRestoreSettings.none(), config.getSavepointRestoreSettings());
+
+		// Set path
+		queryParams.put("savepointPath", "the-savepoint-path");
+		queryParams.put("allowNonRestoredState", "");
+
+		SavepointRestoreSettings expected = SavepointRestoreSettings.forPath("the-savepoint-path", false);
+
+		config = JarActionHandler.JarActionHandlerConfig.fromParams(pathParams, queryParams);
+		assertEquals(expected, config.getSavepointRestoreSettings());
+
+		// Set flag
+		queryParams.put("allowNonRestoredState", "true");
+
+		expected = SavepointRestoreSettings.forPath("the-savepoint-path", true);
+		config = JarActionHandler.JarActionHandlerConfig.fromParams(pathParams, queryParams);
+		assertEquals(expected, config.getSavepointRestoreSettings());
+	}
+
+	/**
+	 * Tests that empty String params are handled ignored.
+	 */
+	@Test
+	public void testEmptyStringParams() throws Exception {
+		Map<String, String> pathParams = new HashMap<>();
+		pathParams.put("jarid", "required"); // required
+		Map<String, String> queryParams = new HashMap<>();
+
+		queryParams.put("program-args", "");
+		queryParams.put("entry-class", "");
+		queryParams.put("parallelism", "");
+		queryParams.put("savepointPath", "");
+		queryParams.put("allowNonRestoredState", "");
+
+		// Nothing configured
+		JarActionHandler.JarActionHandlerConfig config = JarActionHandler.JarActionHandlerConfig.fromParams(pathParams, queryParams);
+
+		assertEquals(0, config.getProgramArgs().length);
+		Assert.assertNull(config.getEntryClass());
+		assertEquals(1, config.getParallelism());
+		assertEquals(SavepointRestoreSettings.none(), config.getSavepointRestoreSettings());
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarDeleteHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarDeleteHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarDeleteHandlerTest.java
new file mode 100644
index 0000000..25c2457
--- /dev/null
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarDeleteHandlerTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers.legacy;
+
+import org.apache.flink.runtime.concurrent.Executors;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests for the JarDeleteHandler.
+ */
+public class JarDeleteHandlerTest {
+	@Test
+	public void testGetPaths() {
+		JarDeleteHandler handler = new JarDeleteHandler(Executors.directExecutor(), null);
+		String[] paths = handler.getPaths();
+		Assert.assertEquals(1, paths.length);
+		Assert.assertEquals("/jars/:jarid", paths[0]);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarListHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarListHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarListHandlerTest.java
new file mode 100644
index 0000000..5cfa2dc
--- /dev/null
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarListHandlerTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers.legacy;
+
+import org.apache.flink.runtime.concurrent.Executors;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests for the JarListHandler.
+ */
+public class JarListHandlerTest {
+	@Test
+	public void testGetPaths() {
+		JarListHandler handler = new JarListHandler(Executors.directExecutor(), null);
+		String[] paths = handler.getPaths();
+		Assert.assertEquals(1, paths.length);
+		Assert.assertEquals("/jars", paths[0]);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarPlanHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarPlanHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarPlanHandlerTest.java
new file mode 100644
index 0000000..273d43c
--- /dev/null
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarPlanHandlerTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers.legacy;
+
+import org.apache.flink.runtime.concurrent.Executors;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests for the JarPlanHandler.
+ */
+public class JarPlanHandlerTest {
+	@Test
+	public void testGetPaths() {
+		JarPlanHandler handler = new JarPlanHandler(Executors.directExecutor(), null);
+		String[] paths = handler.getPaths();
+		Assert.assertEquals(1, paths.length);
+		Assert.assertEquals("/jars/:jarid/plan", paths[0]);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarRunHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarRunHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarRunHandlerTest.java
new file mode 100644
index 0000000..71a1b5a
--- /dev/null
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarRunHandlerTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers.legacy;
+
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.concurrent.Executors;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests for the JarRunHandler.
+ */
+public class JarRunHandlerTest {
+	@Test
+	public void testGetPaths() {
+		JarRunHandler handler = new JarRunHandler(Executors.directExecutor(), null, Time.seconds(0L), new Configuration());
+		String[] paths = handler.getPaths();
+		Assert.assertEquals(1, paths.length);
+		Assert.assertEquals("/jars/:jarid/run", paths[0]);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarUploadHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarUploadHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarUploadHandlerTest.java
new file mode 100644
index 0000000..91941f9
--- /dev/null
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/legacy/JarUploadHandlerTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers.legacy;
+
+import org.apache.flink.runtime.concurrent.Executors;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests for the JarUploadHandler.
+ */
+public class JarUploadHandlerTest {
+	@Test
+	public void testGetPaths() {
+		JarUploadHandler handler = new JarUploadHandler(Executors.directExecutor(), null);
+		String[] paths = handler.getPaths();
+		Assert.assertEquals(1, paths.length);
+		Assert.assertEquals("/jars/upload", paths[0]);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java
index 90022ed..b84e287 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.dispatcher;
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.WebOptions;
 import org.apache.flink.runtime.blob.TransientBlobService;
 import org.apache.flink.runtime.leaderelection.LeaderElectionService;
 import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
@@ -31,11 +32,13 @@ import org.apache.flink.runtime.rest.handler.job.BlobServerPortHandler;
 import org.apache.flink.runtime.rest.handler.job.JobSubmitHandler;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.webmonitor.WebMonitorEndpoint;
+import org.apache.flink.runtime.webmonitor.WebMonitorUtils;
 import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
 import org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever;
 
 import org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandler;
 
+import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
@@ -46,6 +49,8 @@ import java.util.concurrent.Executor;
  */
 public class DispatcherRestEndpoint extends WebMonitorEndpoint<DispatcherGateway> {
 
+	private final Path uploadDir;
+
 	public DispatcherRestEndpoint(
 			RestServerEndpointConfiguration endpointConfiguration,
 			GatewayRetriever<DispatcherGateway> leaderRetriever,
@@ -57,6 +62,7 @@ public class DispatcherRestEndpoint extends WebMonitorEndpoint<DispatcherGateway
 			MetricQueryServiceRetriever metricQueryServiceRetriever,
 			LeaderElectionService leaderElectionService,
 			FatalErrorHandler fatalErrorHandler) {
+
 		super(
 			endpointConfiguration,
 			leaderRetriever,
@@ -68,6 +74,8 @@ public class DispatcherRestEndpoint extends WebMonitorEndpoint<DispatcherGateway
 			metricQueryServiceRetriever,
 			leaderElectionService,
 			fatalErrorHandler);
+
+		uploadDir = endpointConfiguration.getUploadDir();
 	}
 
 	@Override
@@ -91,6 +99,16 @@ public class DispatcherRestEndpoint extends WebMonitorEndpoint<DispatcherGateway
 			timeout,
 			responseHeaders);
 
+		if (clusterConfiguration.getBoolean(WebOptions.SUBMIT_ENABLE)) {
+			handlers.addAll(WebMonitorUtils.tryLoadJarHandlers(
+				leaderRetriever,
+				restAddressFuture,
+				timeout,
+				responseHeaders,
+				uploadDir,
+				executor));
+		}
+
 		handlers.add(Tuple2.of(blobServerPortHandler.getMessageHeaders(), blobServerPortHandler));
 		handlers.add(Tuple2.of(jobSubmitHandler.getMessageHeaders(), jobSubmitHandler));
 

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime/src/main/java/org/apache/flink/runtime/rest/AbstractHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/AbstractHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/AbstractHandler.java
index 2510429..a259801 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/AbstractHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/AbstractHandler.java
@@ -25,6 +25,7 @@ import org.apache.flink.runtime.rest.handler.RedirectHandler;
 import org.apache.flink.runtime.rest.handler.RestHandlerException;
 import org.apache.flink.runtime.rest.handler.util.HandlerUtils;
 import org.apache.flink.runtime.rest.messages.ErrorResponseBody;
+import org.apache.flink.runtime.rest.messages.FileUpload;
 import org.apache.flink.runtime.rest.messages.MessageParameters;
 import org.apache.flink.runtime.rest.messages.RequestBody;
 import org.apache.flink.runtime.rest.messages.UntypedResponseMessageHeaders;
@@ -49,6 +50,7 @@ import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nonnull;
 
+import java.nio.file.Path;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 
@@ -105,7 +107,20 @@ public abstract class AbstractHandler<T extends RestfulGateway, R extends Reques
 			ByteBuf msgContent = ((FullHttpRequest) httpRequest).content();
 
 			R request;
-			if (msgContent.capacity() == 0) {
+			if (isFileUpload()) {
+				final Path path = ctx.channel().attr(FileUploadHandler.UPLOADED_FILE).get();
+				if (path == null) {
+					HandlerUtils.sendErrorResponse(
+						ctx,
+						httpRequest,
+						new ErrorResponseBody("Client did not upload a file."),
+						HttpResponseStatus.BAD_REQUEST,
+						responseHeaders);
+					return;
+				}
+				//noinspection unchecked
+				request = (R) new FileUpload(path);
+			} else if (msgContent.capacity() == 0) {
 				try {
 					request = MAPPER.readValue("{}", untypedResponseMessageHeaders.getRequestClass());
 				} catch (JsonParseException | JsonMappingException je) {
@@ -167,6 +182,10 @@ public abstract class AbstractHandler<T extends RestfulGateway, R extends Reques
 		}
 	}
 
+	private boolean isFileUpload() {
+		return untypedResponseMessageHeaders.getRequestClass() == FileUpload.class;
+	}
+
 	/**
 	 * Respond to the given {@link HandlerRequest}.
 	 *

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FileUploadHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FileUploadHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FileUploadHandler.java
new file mode 100644
index 0000000..c214b50
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FileUploadHandler.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rest;
+
+import org.apache.flink.annotation.VisibleForTesting;
+
+import org.apache.flink.shaded.netty4.io.netty.channel.ChannelHandlerContext;
+import org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandler;
+import org.apache.flink.shaded.netty4.io.netty.channel.ChannelPipeline;
+import org.apache.flink.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler;
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpContent;
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpMethod;
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpObject;
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpRequest;
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.LastHttpContent;
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.multipart.DefaultHttpDataFactory;
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.multipart.DiskFileUpload;
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.multipart.HttpDataFactory;
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.multipart.HttpPostRequestDecoder;
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.multipart.InterfaceHttpData;
+import org.apache.flink.shaded.netty4.io.netty.util.AttributeKey;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.UUID;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
+/**
+ * Writes multipart/form-data to disk. Delegates all other requests to the next
+ * {@link ChannelInboundHandler} in the {@link ChannelPipeline}.
+ */
+public class FileUploadHandler extends SimpleChannelInboundHandler<HttpObject> {
+
+	private static final Logger LOG = LoggerFactory.getLogger(FileUploadHandler.class);
+
+	static final AttributeKey<Path> UPLOADED_FILE = AttributeKey.valueOf("UPLOADED_FILE");
+
+	private static final HttpDataFactory DATA_FACTORY = new DefaultHttpDataFactory(true);
+
+	private final Path uploadDir;
+
+	private HttpPostRequestDecoder currentHttpPostRequestDecoder;
+
+	private HttpRequest currentHttpRequest;
+
+	public FileUploadHandler(final Path uploadDir) throws IOException {
+		super(false);
+		createUploadDir(uploadDir);
+
+		DiskFileUpload.baseDirectory = uploadDir.normalize().toAbsolutePath().toString();
+		this.uploadDir = requireNonNull(uploadDir);
+	}
+
+	@Override
+	protected void channelRead0(final ChannelHandlerContext ctx, final HttpObject msg) throws Exception {
+		if (msg instanceof HttpRequest) {
+			final HttpRequest httpRequest = (HttpRequest) msg;
+			if (httpRequest.getMethod().equals(HttpMethod.POST)) {
+				if (HttpPostRequestDecoder.isMultipart(httpRequest)) {
+					currentHttpPostRequestDecoder = new HttpPostRequestDecoder(DATA_FACTORY, httpRequest);
+					currentHttpRequest = httpRequest;
+				} else {
+					ctx.fireChannelRead(msg);
+				}
+			} else {
+				ctx.fireChannelRead(msg);
+			}
+		} else if (msg instanceof HttpContent && currentHttpPostRequestDecoder != null) {
+			createUploadDir(uploadDir);
+
+			final HttpContent httpContent = (HttpContent) msg;
+			currentHttpPostRequestDecoder.offer(httpContent);
+
+			while (currentHttpPostRequestDecoder.hasNext()) {
+				final InterfaceHttpData data = currentHttpPostRequestDecoder.next();
+				if (data.getHttpDataType() == InterfaceHttpData.HttpDataType.FileUpload) {
+					final DiskFileUpload fileUpload = (DiskFileUpload) data;
+					checkState(fileUpload.isCompleted());
+
+					final Path dest = uploadDir.resolve(Paths.get(UUID.randomUUID() +
+						"_" + fileUpload.getFilename()));
+					fileUpload.renameTo(dest.toFile());
+					ctx.channel().attr(UPLOADED_FILE).set(dest);
+				}
+				data.release();
+			}
+
+			if (httpContent instanceof LastHttpContent) {
+				ctx.fireChannelRead(currentHttpRequest);
+				ctx.fireChannelRead(httpContent);
+				reset();
+			}
+		} else {
+			ctx.fireChannelRead(msg);
+		}
+	}
+
+	private void reset() {
+		currentHttpPostRequestDecoder.destroy();
+		currentHttpPostRequestDecoder = null;
+		currentHttpRequest = null;
+	}
+
+	/**
+	 * Creates the upload dir if needed.
+	 */
+	@VisibleForTesting
+	static void createUploadDir(final Path uploadDir) throws IOException {
+		if (!Files.exists(uploadDir)) {
+			LOG.warn("Upload directory {} does not exist, or has been deleted externally. " +
+				"Previously uploaded files are no longer available.", uploadDir);
+			checkAndCreateUploadDir(uploadDir);
+		}
+	}
+
+	/**
+	 * Checks whether the given directory exists and is writable. If it doesn't exist, this method
+	 * will attempt to create it.
+	 *
+	 * @param uploadDir directory to check
+	 * @throws IOException if the directory does not exist and cannot be created, or if the
+	 *                     directory isn't writable
+	 */
+	private static synchronized void checkAndCreateUploadDir(final Path uploadDir) throws IOException {
+		if (Files.exists(uploadDir) && Files.isWritable(uploadDir)) {
+			LOG.info("Using directory {} for file uploads.", uploadDir);
+		} else if (Files.isWritable(Files.createDirectories(uploadDir))) {
+			LOG.info("Created directory {} for file uploads.", uploadDir);
+		} else {
+			LOG.warn("Upload directory {} cannot be created or is not writable.", uploadDir);
+			throw new IOException(
+				String.format("Upload directory %s cannot be created or is not writable.",
+					uploadDir));
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java
index a5a7473..1904235 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java
@@ -23,6 +23,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.runtime.rest.handler.PipelineErrorHandler;
 import org.apache.flink.runtime.rest.handler.RestHandlerSpecification;
 import org.apache.flink.runtime.rest.handler.RouterHandler;
+import org.apache.flink.util.FileUtils;
 import org.apache.flink.util.Preconditions;
 
 import org.apache.flink.shaded.netty4.io.netty.bootstrap.ServerBootstrap;
@@ -45,8 +46,10 @@ import org.slf4j.LoggerFactory;
 
 import javax.net.ssl.SSLEngine;
 
+import java.io.IOException;
 import java.io.Serializable;
 import java.net.InetSocketAddress;
+import java.nio.file.Path;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
@@ -66,6 +69,7 @@ public abstract class RestServerEndpoint {
 	private final String configuredAddress;
 	private final int configuredPort;
 	private final SSLEngine sslEngine;
+	private final Path uploadDir;
 
 	private ServerBootstrap bootstrap;
 	private Channel serverChannel;
@@ -78,6 +82,7 @@ public abstract class RestServerEndpoint {
 		this.configuredAddress = configuration.getEndpointBindAddress();
 		this.configuredPort = configuration.getEndpointBindPort();
 		this.sslEngine = configuration.getSslEngine();
+		this.uploadDir = configuration.getUploadDir();
 
 		this.restAddress = null;
 
@@ -131,7 +136,7 @@ public abstract class RestServerEndpoint {
 			ChannelInitializer<SocketChannel> initializer = new ChannelInitializer<SocketChannel>() {
 
 				@Override
-				protected void initChannel(SocketChannel ch) {
+				protected void initChannel(SocketChannel ch) throws IOException {
 					Handler handler = new RouterHandler(router);
 
 					// SSL should be the first handler in the pipeline
@@ -141,6 +146,7 @@ public abstract class RestServerEndpoint {
 
 					ch.pipeline()
 						.addLast(new HttpServerCodec())
+						.addLast(new FileUploadHandler(uploadDir))
 						.addLast(new HttpObjectAggregator(MAX_REQUEST_SIZE_BYTES))
 						.addLast(handler.name(), handler)
 						.addLast(new PipelineErrorHandler(log));
@@ -282,6 +288,13 @@ public abstract class RestServerEndpoint {
 
 			restAddress = null;
 			started = false;
+
+			try {
+				log.info("Cleaning upload directory {}", uploadDir);
+				FileUtils.cleanDirectory(uploadDir.toFile());
+			} catch (IOException e) {
+				log.warn("Error while cleaning upload directory {}", uploadDir, e);
+			}
 		}
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java
index f342a01..c411b51 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.rest;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.RestOptions;
 import org.apache.flink.configuration.SecurityOptions;
+import org.apache.flink.configuration.WebOptions;
 import org.apache.flink.runtime.net.SSLUtils;
 import org.apache.flink.util.ConfigurationException;
 import org.apache.flink.util.Preconditions;
@@ -29,6 +30,12 @@ import javax.annotation.Nullable;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
 
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.UUID;
+
+import static java.util.Objects.requireNonNull;
+
 /**
  * A configuration object for {@link RestServerEndpoint}s.
  */
@@ -36,16 +43,26 @@ public final class RestServerEndpointConfiguration {
 
 	@Nullable
 	private final String restBindAddress;
+
 	private final int restBindPort;
+
 	@Nullable
 	private final SSLEngine sslEngine;
 
-	private RestServerEndpointConfiguration(@Nullable String restBindAddress, int restBindPort, @Nullable SSLEngine sslEngine) {
-		this.restBindAddress = restBindAddress;
+	private final Path uploadDir;
+
+	private RestServerEndpointConfiguration(
+			@Nullable String restBindAddress,
+			int restBindPort,
+			@Nullable SSLEngine sslEngine,
+			final Path uploadDir) {
 
 		Preconditions.checkArgument(0 <= restBindPort && restBindPort < 65536, "The bing rest port " + restBindPort + " is out of range (0, 65536[");
+
+		this.restBindAddress = restBindAddress;
 		this.restBindPort = restBindPort;
 		this.sslEngine = sslEngine;
+		this.uploadDir = requireNonNull(uploadDir);
 	}
 
 	/**
@@ -76,6 +93,13 @@ public final class RestServerEndpointConfiguration {
 	}
 
 	/**
+	 * Returns the directory used to temporarily store multipart/form-data uploads.
+	 */
+	public Path getUploadDir() {
+		return uploadDir;
+	}
+
+	/**
 	 * Creates and returns a new {@link RestServerEndpointConfiguration} from the given {@link Configuration}.
 	 *
 	 * @param config configuration from which the REST server endpoint configuration should be created from
@@ -103,6 +127,10 @@ public final class RestServerEndpointConfiguration {
 			}
 		}
 
-		return new RestServerEndpointConfiguration(address, port, sslEngine);
+		final Path uploadDir = Paths.get(
+			config.getString(WebOptions.UPLOAD_DIR,	config.getString(WebOptions.TMP_DIR)),
+			"flink-web-upload-" + UUID.randomUUID());
+
+		return new RestServerEndpointConfiguration(address, port, sslEngine, uploadDir);
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/FileUpload.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/FileUpload.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/FileUpload.java
new file mode 100644
index 0000000..fb09dce
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/FileUpload.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rest.messages;
+
+import java.nio.file.Path;
+
+/**
+ * Client uploading a file.
+ */
+public class FileUpload implements RequestBody {
+
+	private final Path path;
+
+	public FileUpload(final Path path) {
+		this.path = path;
+	}
+
+	public Path getPath() {
+		return path;
+	}
+
+	@Override
+	public String toString() {
+		return "FileUpload{" +
+			"path=" + path +
+			'}';
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
index cee02fe..b148d36 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
@@ -136,11 +136,11 @@ import java.util.concurrent.Executor;
 public class WebMonitorEndpoint<T extends RestfulGateway> extends RestServerEndpoint implements LeaderContender {
 
 	protected final GatewayRetriever<? extends T> leaderRetriever;
-	private final Configuration clusterConfiguration;
+	protected final Configuration clusterConfiguration;
 	protected final RestHandlerConfiguration restConfiguration;
 	private final GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever;
 	private final TransientBlobService transientBlobService;
-	private final Executor executor;
+	protected final Executor executor;
 
 	private final ExecutionGraphCache executionGraphCache;
 	private final CheckpointStatsCache checkpointStatsCache;

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
index a603340..3b4da84 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java
@@ -19,6 +19,7 @@
 package org.apache.flink.runtime.webmonitor;
 
 import org.apache.flink.api.common.time.Time;
+import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.WebOptions;
 import org.apache.flink.core.fs.Path;
@@ -32,7 +33,9 @@ import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.jobmaster.JobManagerGateway;
 import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
 import org.apache.flink.runtime.messages.webmonitor.JobDetails;
+import org.apache.flink.runtime.rest.handler.RestHandlerSpecification;
 import org.apache.flink.runtime.rest.handler.legacy.files.StaticFileServerHandler;
+import org.apache.flink.runtime.rest.messages.MessageHeaders;
 import org.apache.flink.runtime.webmonitor.history.JsonArchivist;
 import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
 import org.apache.flink.runtime.webmonitor.retriever.LeaderGatewayRetriever;
@@ -41,6 +44,7 @@ import org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever
 import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
 import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode;
+import org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandler;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -51,11 +55,15 @@ import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.net.URI;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
 
 /**
  * Utilities for the web runtime monitor. This class contains for example methods to build
@@ -64,6 +72,8 @@ import java.util.concurrent.CompletableFuture;
  */
 public final class WebMonitorUtils {
 
+	private static final String WEB_RUNTIME_MONITOR_CLASS_FQN = "org.apache.flink.runtime.webmonitor.WebRuntimeMonitor";
+
 	private static final Logger LOG = LoggerFactory.getLogger(WebMonitorUtils.class);
 
 	/**
@@ -142,8 +152,7 @@ public final class WebMonitorUtils {
 			ScheduledExecutor scheduledExecutor) {
 		// try to load and instantiate the class
 		try {
-			String classname = "org.apache.flink.runtime.webmonitor.WebRuntimeMonitor";
-			Class<? extends WebMonitor> clazz = Class.forName(classname).asSubclass(WebMonitor.class);
+			Class<? extends WebMonitor> clazz = Class.forName(WEB_RUNTIME_MONITOR_CLASS_FQN).asSubclass(WebMonitor.class);
 
 			Constructor<? extends WebMonitor> constructor = clazz.getConstructor(
 				Configuration.class,
@@ -191,29 +200,65 @@ public final class WebMonitorUtils {
 			Time timeout,
 			File tmpDir) throws IOException {
 
-		// 1. Check if flink-runtime-web is in the classpath
-		try {
-			final String classname = "org.apache.flink.runtime.webmonitor.WebRuntimeMonitor";
-			Class.forName(classname).asSubclass(WebMonitor.class);
-
+		if (isFlinkRuntimeWebInClassPath()) {
 			return Optional.of(new StaticFileServerHandler<>(
 				leaderRetriever,
 				restAddressFuture,
 				timeout,
 				tmpDir));
-		} catch (ClassNotFoundException ignored) {
-			// class not found means that there is no flink-runtime-web in the classpath
+		} else {
 			return Optional.empty();
 		}
 	}
 
+	@SuppressWarnings({"unchecked", "rawtypes"})
+	public static <T extends RestfulGateway> Collection<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> tryLoadJarHandlers(
+			GatewayRetriever<T> leaderRetriever,
+			CompletableFuture<String> restAddressFuture,
+			Time timeout,
+			Map<String, String> responseHeaders,
+			java.nio.file.Path uploadDir,
+			Executor executor) {
+
+		if (!isFlinkRuntimeWebInClassPath()) {
+			return Collections.emptyList();
+		}
+
+		try {
+			final String classname = "org.apache.flink.runtime.webmonitor.handlers.JarUploadHandler";
+			final Class<?> clazz = Class.forName(classname);
+			final Constructor<?> constructor = clazz.getConstructor(
+				CompletableFuture.class,
+				GatewayRetriever.class,
+				Time.class,
+				Map.class,
+				MessageHeaders.class,
+				java.nio.file.Path.class,
+				Executor.class);
+
+			final MessageHeaders jarUploadMessageHeaders =
+				(MessageHeaders) Class
+					.forName("org.apache.flink.runtime.webmonitor.handlers.JarUploadMessageHeaders")
+					.newInstance();
+
+			return Arrays.asList(Tuple2.of(jarUploadMessageHeaders, (ChannelInboundHandler) constructor.newInstance(
+				restAddressFuture,
+				leaderRetriever,
+				timeout,
+				responseHeaders,
+				jarUploadMessageHeaders,
+				uploadDir,
+				executor)));
+		} catch (ClassNotFoundException | InvocationTargetException | InstantiationException | NoSuchMethodException | IllegalAccessException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
 	public static JsonArchivist[] getJsonArchivists() {
 		try {
-			String classname = "org.apache.flink.runtime.webmonitor.WebRuntimeMonitor";
-			Class<? extends WebMonitor> clazz = Class.forName(classname).asSubclass(WebMonitor.class);
+			Class<? extends WebMonitor> clazz = Class.forName(WEB_RUNTIME_MONITOR_CLASS_FQN).asSubclass(WebMonitor.class);
 			Method method = clazz.getMethod("getJsonArchivists");
-			JsonArchivist[] result = (JsonArchivist[]) method.invoke(null);
-			return result;
+			return (JsonArchivist[]) method.invoke(null);
 		} catch (ClassNotFoundException e) {
 			LOG.error("Could not load web runtime monitor. " +
 				"Probably reason: flink-runtime-web is not in the classpath");
@@ -314,4 +359,18 @@ public final class WebMonitorUtils {
 	private WebMonitorUtils() {
 		throw new RuntimeException();
 	}
+
+	/**
+	 * Returns {@code true} if the optional dependency {@code flink-runtime-web} is in the
+	 * classpath.
+	 */
+	private static boolean isFlinkRuntimeWebInClassPath() {
+		try {
+			Class.forName(WEB_RUNTIME_MONITOR_CLASS_FQN).asSubclass(WebMonitor.class);
+			return true;
+		} catch (ClassNotFoundException e) {
+			// class not found means that there is no flink-runtime-web in the classpath
+			return false;
+		}
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java
new file mode 100644
index 0000000..61277ae
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rest;
+
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests for {@link FileUploadHandler}.
+ */
+public class FileUploadHandlerTest extends TestLogger {
+
+	@Rule
+	public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+	@Test
+	public void testCreateUploadDir() throws Exception {
+		final File file = temporaryFolder.newFolder();
+		final Path testUploadDir = file.toPath().resolve("testUploadDir");
+		assertFalse(Files.exists(testUploadDir));
+		FileUploadHandler.createUploadDir(testUploadDir);
+		assertTrue(Files.exists(testUploadDir));
+	}
+
+	@Test
+	public void testCreateUploadDirFails() throws Exception {
+		final File file = temporaryFolder.newFolder();
+		Assume.assumeTrue(file.setWritable(false));
+
+		final Path testUploadDir = file.toPath().resolve("testUploadDir");
+		assertFalse(Files.exists(testUploadDir));
+		try {
+			FileUploadHandler.createUploadDir(testUploadDir);
+			fail("Expected exception not thrown.");
+		} catch (IOException e) {
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/39df56d6/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestEndpointITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestEndpointITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestEndpointITCase.java
deleted file mode 100644
index 1cdc70a..0000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestEndpointITCase.java
+++ /dev/null
@@ -1,383 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.rest;
-
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.time.Time;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.RestOptions;
-import org.apache.flink.runtime.rest.handler.AbstractRestHandler;
-import org.apache.flink.runtime.rest.handler.HandlerRequest;
-import org.apache.flink.runtime.rest.handler.RestHandlerException;
-import org.apache.flink.runtime.rest.handler.RestHandlerSpecification;
-import org.apache.flink.runtime.rest.messages.ConversionException;
-import org.apache.flink.runtime.rest.messages.MessageHeaders;
-import org.apache.flink.runtime.rest.messages.MessageParameters;
-import org.apache.flink.runtime.rest.messages.MessagePathParameter;
-import org.apache.flink.runtime.rest.messages.MessageQueryParameter;
-import org.apache.flink.runtime.rest.messages.RequestBody;
-import org.apache.flink.runtime.rest.messages.ResponseBody;
-import org.apache.flink.runtime.rest.util.RestClientException;
-import org.apache.flink.runtime.rpc.RpcUtils;
-import org.apache.flink.runtime.testingUtils.TestingUtils;
-import org.apache.flink.runtime.webmonitor.RestfulGateway;
-import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
-import org.apache.flink.testutils.category.Flip6;
-import org.apache.flink.util.ExceptionUtils;
-import org.apache.flink.util.Preconditions;
-import org.apache.flink.util.TestLogger;
-
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandler;
-import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import javax.annotation.Nonnull;
-
-import java.net.InetSocketAddress;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * IT cases for {@link RestClient} and {@link RestServerEndpoint}.
- */
-@Category(Flip6.class)
-public class RestEndpointITCase extends TestLogger {
-
-	private static final JobID PATH_JOB_ID = new JobID();
-	private static final JobID QUERY_JOB_ID = new JobID();
-	private static final String JOB_ID_KEY = "jobid";
-	private static final Time timeout = Time.seconds(10L);
-
-	private RestServerEndpoint serverEndpoint;
-	private RestClient restClient;
-
-	@Before
-	public void setup() throws Exception {
-		Configuration config = new Configuration();
-		config.setInteger(RestOptions.REST_PORT, 0);
-
-		RestServerEndpointConfiguration serverConfig = RestServerEndpointConfiguration.fromConfiguration(config);
-		RestClientConfiguration clientConfig = RestClientConfiguration.fromConfiguration(config);
-
-		final String restAddress = "http://localhost:1234";
-		RestfulGateway mockRestfulGateway = mock(RestfulGateway.class);
-		when(mockRestfulGateway.requestRestAddress(any(Time.class))).thenReturn(CompletableFuture.completedFuture(restAddress));
-		GatewayRetriever<RestfulGateway> mockGatewayRetriever = mock(GatewayRetriever.class);
-		when(mockGatewayRetriever.getNow()).thenReturn(Optional.of(mockRestfulGateway));
-
-		TestHandler testHandler = new TestHandler(
-			CompletableFuture.completedFuture(restAddress),
-			mockGatewayRetriever,
-			RpcUtils.INF_TIMEOUT);
-
-		serverEndpoint = new TestRestServerEndpoint(serverConfig, testHandler);
-		restClient = new TestRestClient(clientConfig);
-
-		serverEndpoint.start();
-	}
-
-	@After
-	public void teardown() {
-		if (restClient != null) {
-			restClient.shutdown(timeout);
-			restClient = null;
-		}
-
-		if (serverEndpoint != null) {
-			serverEndpoint.shutdown(timeout);
-			serverEndpoint = null;
-		}
-	}
-
-	/**
-	 * Tests that request are handled as individual units which don't interfere with each other.
-	 * This means that request responses can overtake each other.
-	 */
-	@Test
-	public void testRequestInterleaving() throws Exception {
-
-		TestParameters parameters = new TestParameters();
-		parameters.jobIDPathParameter.resolve(PATH_JOB_ID);
-		parameters.jobIDQueryParameter.resolve(Collections.singletonList(QUERY_JOB_ID));
-
-		// send first request and wait until the handler blocks
-		CompletableFuture<TestResponse> response1;
-		final InetSocketAddress serverAddress = serverEndpoint.getServerAddress();
-
-		synchronized (TestHandler.LOCK) {
-			response1 = restClient.sendRequest(
-				serverAddress.getHostName(),
-				serverAddress.getPort(),
-				new TestHeaders(),
-				parameters,
-				new TestRequest(1));
-			TestHandler.LOCK.wait();
-		}
-
-		// send second request and verify response
-		CompletableFuture<TestResponse> response2 = restClient.sendRequest(
-			serverAddress.getHostName(),
-			serverAddress.getPort(),
-			new TestHeaders(),
-			parameters,
-			new TestRequest(2));
-		Assert.assertEquals(2, response2.get().id);
-
-		// wake up blocked handler
-		synchronized (TestHandler.LOCK) {
-			TestHandler.LOCK.notifyAll();
-		}
-		// verify response to first request
-		Assert.assertEquals(1, response1.get().id);
-	}
-
-	/**
-	 * Tests that a bad handler request (HandlerRequest cannot be created) is reported as a BAD_REQUEST
-	 * and not an internal server error.
-	 *
-	 * <p>See FLINK-7663
-	 */
-	@Test
-	public void testBadHandlerRequest() throws Exception {
-		final InetSocketAddress serverAddress = serverEndpoint.getServerAddress();
-
-		final FaultyTestParameters parameters = new FaultyTestParameters();
-
-		parameters.faultyJobIDPathParameter.resolve(PATH_JOB_ID);
-		((TestParameters) parameters).jobIDQueryParameter.resolve(Collections.singletonList(QUERY_JOB_ID));
-
-		CompletableFuture<TestResponse> response = restClient.sendRequest(
-			serverAddress.getHostName(),
-			serverAddress.getPort(),
-			new TestHeaders(),
-			parameters,
-			new TestRequest(2));
-
-		try {
-			response.get();
-
-			Assert.fail("The request should fail with a bad request return code.");
-		} catch (ExecutionException ee) {
-			Throwable t = ExceptionUtils.stripExecutionException(ee);
-
-			Assert.assertTrue(t instanceof RestClientException);
-
-			RestClientException rce = (RestClientException) t;
-
-			Assert.assertEquals(HttpResponseStatus.BAD_REQUEST, rce.getHttpResponseStatus());
-		}
-	}
-
-	private static class TestRestServerEndpoint extends RestServerEndpoint {
-
-		private final TestHandler testHandler;
-
-		TestRestServerEndpoint(RestServerEndpointConfiguration configuration, TestHandler testHandler) {
-			super(configuration);
-
-			this.testHandler = Preconditions.checkNotNull(testHandler);
-		}
-
-		@Override
-		protected List<Tuple2<RestHandlerSpecification, ChannelInboundHandler>> initializeHandlers(CompletableFuture<String> restAddressFuture) {
-			return Collections.singletonList(Tuple2.of(new TestHeaders(), testHandler));
-		}
-	}
-
-	private static class TestHandler extends AbstractRestHandler<RestfulGateway, TestRequest, TestResponse, TestParameters> {
-
-		public static final Object LOCK = new Object();
-
-		TestHandler(
-			CompletableFuture<String> localAddressFuture,
-			GatewayRetriever<RestfulGateway> leaderRetriever,
-			Time timeout) {
-			super(
-				localAddressFuture,
-				leaderRetriever,
-				timeout,
-				Collections.emptyMap(),
-				new TestHeaders());
-		}
-
-		@Override
-		protected CompletableFuture<TestResponse> handleRequest(@Nonnull HandlerRequest<TestRequest, TestParameters> request, RestfulGateway gateway) throws RestHandlerException {
-			Assert.assertEquals(request.getPathParameter(JobIDPathParameter.class), PATH_JOB_ID);
-			Assert.assertEquals(request.getQueryParameter(JobIDQueryParameter.class).get(0), QUERY_JOB_ID);
-
-			if (request.getRequestBody().id == 1) {
-				synchronized (LOCK) {
-					try {
-						LOCK.notifyAll();
-						LOCK.wait();
-					} catch (InterruptedException ignored) {
-					}
-				}
-			}
-			return CompletableFuture.completedFuture(new TestResponse(request.getRequestBody().id));
-		}
-	}
-
-	private static class TestRestClient extends RestClient {
-
-		TestRestClient(RestClientConfiguration configuration) {
-			super(configuration, TestingUtils.defaultExecutor());
-		}
-	}
-
-	private static class TestRequest implements RequestBody {
-		public final int id;
-
-		@JsonCreator
-		public TestRequest(@JsonProperty("id") int id) {
-			this.id = id;
-		}
-	}
-
-	private static class TestResponse implements ResponseBody {
-		public final int id;
-
-		@JsonCreator
-		public TestResponse(@JsonProperty("id") int id) {
-			this.id = id;
-		}
-	}
-
-	private static class TestHeaders implements MessageHeaders<TestRequest, TestResponse, TestParameters> {
-
-		@Override
-		public HttpMethodWrapper getHttpMethod() {
-			return HttpMethodWrapper.POST;
-		}
-
-		@Override
-		public String getTargetRestEndpointURL() {
-			return "/test/:jobid";
-		}
-
-		@Override
-		public Class<TestRequest> getRequestClass() {
-			return TestRequest.class;
-		}
-
-		@Override
-		public Class<TestResponse> getResponseClass() {
-			return TestResponse.class;
-		}
-
-		@Override
-		public HttpResponseStatus getResponseStatusCode() {
-			return HttpResponseStatus.OK;
-		}
-
-		@Override
-		public TestParameters getUnresolvedMessageParameters() {
-			return new TestParameters();
-		}
-	}
-
-	private static class TestParameters extends MessageParameters {
-		private final JobIDPathParameter jobIDPathParameter = new JobIDPathParameter();
-		private final JobIDQueryParameter jobIDQueryParameter = new JobIDQueryParameter();
-
-		@Override
-		public Collection<MessagePathParameter<?>> getPathParameters() {
-			return Collections.singleton(jobIDPathParameter);
-		}
-
-		@Override
-		public Collection<MessageQueryParameter<?>> getQueryParameters() {
-			return Collections.singleton(jobIDQueryParameter);
-		}
-	}
-
-	private static class FaultyTestParameters extends TestParameters {
-		private final FaultyJobIDPathParameter faultyJobIDPathParameter = new FaultyJobIDPathParameter();
-
-		@Override
-		public Collection<MessagePathParameter<?>> getPathParameters() {
-			return Collections.singleton(faultyJobIDPathParameter);
-		}
-	}
-
-	static class JobIDPathParameter extends MessagePathParameter<JobID> {
-		JobIDPathParameter() {
-			super(JOB_ID_KEY);
-		}
-
-		@Override
-		public JobID convertFromString(String value) {
-			return JobID.fromHexString(value);
-		}
-
-		@Override
-		protected String convertToString(JobID value) {
-			return value.toString();
-		}
-	}
-
-	static class FaultyJobIDPathParameter extends MessagePathParameter<JobID> {
-
-		FaultyJobIDPathParameter() {
-			super(JOB_ID_KEY);
-		}
-
-		@Override
-		protected JobID convertFromString(String value) throws ConversionException {
-			return JobID.fromHexString(value);
-		}
-
-		@Override
-		protected String convertToString(JobID value) {
-			return "foobar";
-		}
-	}
-
-	static class JobIDQueryParameter extends MessageQueryParameter<JobID> {
-		JobIDQueryParameter() {
-			super(JOB_ID_KEY, MessageParameterRequisiteness.MANDATORY);
-		}
-
-		@Override
-		public JobID convertValueFromString(String value) {
-			return JobID.fromHexString(value);
-		}
-
-		@Override
-		public String convertStringToValue(JobID value) {
-			return value.toString();
-		}
-	}
-}


[06/22] flink git commit: [FLINK-8605] [rest] Enable job cancellation from the web UI

Posted by tr...@apache.org.
[FLINK-8605] [rest] Enable job cancellation from the web UI

In order to support the job cancellation from the web UI, including when using
Yarn, we have to register the JobTerminationHandler under /jobs/:jobid/yarn-cancel
and /jobs/:jobid/yarn-stop. This is just a temporary fix until we can send
arbitrary REST verbs through the Yarn proxy.

This closes #5430.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2a154173
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2a154173
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2a154173

Branch: refs/heads/master
Commit: 2a154173c7914901a6df5b99ff760f1417159bfc
Parents: a510589
Author: Till Rohrmann <tr...@apache.org>
Authored: Wed Feb 7 12:35:58 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Thu Feb 15 08:44:18 2018 +0100

----------------------------------------------------------------------
 .../app/scripts/modules/jobs/jobs.svc.coffee    |  4 +-
 .../web-dashboard/web/js/hs/index.js            |  4 +-
 flink-runtime-web/web-dashboard/web/js/index.js |  4 +-
 .../rest/handler/job/JobTerminationHandler.java | 10 +++-
 .../YarnCancelJobTerminationHeaders.java        | 55 ++++++++++++++++++++
 .../messages/YarnStopJobTerminationHeaders.java | 55 ++++++++++++++++++++
 .../runtime/webmonitor/WebMonitorEndpoint.java  | 22 ++++++--
 .../YarnCancelJobTerminationHeadersTest.java    | 47 +++++++++++++++++
 .../YarnStopJobTerminationHeadersTest.java      | 48 +++++++++++++++++
 9 files changed, 238 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.svc.coffee
----------------------------------------------------------------------
diff --git a/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.svc.coffee b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.svc.coffee
index 5441730..b555430 100644
--- a/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.svc.coffee
+++ b/flink-runtime-web/web-dashboard/app/scripts/modules/jobs/jobs.svc.coffee
@@ -314,12 +314,12 @@ angular.module('flinkApp')
 
   @cancelJob = (jobid) ->
     # uses the non REST-compliant GET yarn-cancel handler which is available in addition to the
-    # proper "DELETE jobs/<jobid>/"
+    # proper $http.patch flinkConfig.jobServer + "jobs/" + jobid + "?mode=cancel"
     $http.get flinkConfig.jobServer + "jobs/" + jobid + "/yarn-cancel"
 
   @stopJob = (jobid) ->
     # uses the non REST-compliant GET yarn-cancel handler which is available in addition to the
-    # proper "DELETE jobs/<jobid>/"
+    # proper $http.patch flinkConfig.jobServer + "jobs/" + jobid + "?mode=stop"
     $http.get "jobs/" + jobid + "/yarn-stop"
 
   @


[17/22] flink git commit: [FLINK-8530] [flip6] Enable detached job mode submission to session cluster

Posted by tr...@apache.org.
[FLINK-8530] [flip6] Enable detached job mode submission to session cluster

This commit makes the RestClusterClient aware whether the user wishes to submit
a job in detached or non-detached mode. If it is detached, then the RestClusterClient
won't poll for the execution result.

This closes #5466.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/f57b9833
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/f57b9833
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/f57b9833

Branch: refs/heads/master
Commit: f57b9833aaf7649f79b72efd4a033fbfb4f8e218
Parents: 39df56d
Author: Till Rohrmann <tr...@apache.org>
Authored: Mon Feb 12 16:33:15 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Thu Feb 15 08:45:18 2018 +0100

----------------------------------------------------------------------
 .../client/program/rest/RestClusterClient.java  | 67 +++++++++++---------
 .../program/rest/RestClusterClientTest.java     | 49 ++++++++++----
 2 files changed, 74 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/f57b9833/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
index 18cd5c0..2f53545 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
@@ -96,7 +96,6 @@ import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -197,40 +196,46 @@ public class RestClusterClient<T> extends ClusterClient<T> {
 
 	@Override
 	protected JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException {
-		log.info("Submitting job.");
-		try {
-			submitJob(jobGraph).get();
-		} catch (InterruptedException | ExecutionException e) {
-			throw new ProgramInvocationException(ExceptionUtils.stripExecutionException(e));
-		}
+		log.info("Submitting job {}.", jobGraph.getJobID());
 
-		final CompletableFuture<JobResult> jobResultFuture = requestJobResult(jobGraph.getJobID());
+		final CompletableFuture<JobSubmitResponseBody> jobSubmissionFuture = submitJob(jobGraph);
 
-		final JobResult jobResult;
-		try {
-			jobResult = jobResultFuture.get();
-		} catch (Exception e) {
-			throw new ProgramInvocationException(e);
-		}
+		if (isDetached()) {
+			try {
+				jobSubmissionFuture.get();
+			} catch (Exception e) {
+				throw new ProgramInvocationException("Could not submit job " + jobGraph.getJobID() + '.', ExceptionUtils.stripExecutionException(e));
+			}
 
-		if (jobResult.getSerializedThrowable().isPresent()) {
-			final SerializedThrowable serializedThrowable = jobResult.getSerializedThrowable().get();
-			final Throwable throwable = serializedThrowable.deserializeError(classLoader);
-			throw new ProgramInvocationException(throwable);
-		}
+			return new JobSubmissionResult(jobGraph.getJobID());
+		} else {
+			final CompletableFuture<JobResult> jobResultFuture = jobSubmissionFuture.thenCompose(
+				ignored -> requestJobResult(jobGraph.getJobID()));
 
-		try {
-			// don't return just a JobSubmissionResult here, the signature is lying
-			// The CliFrontend expects this to be a JobExecutionResult
-			this.lastJobExecutionResult = new JobExecutionResult(
-				jobResult.getJobId(),
-				jobResult.getNetRuntime(),
-				AccumulatorHelper.deserializeAccumulators(
-					jobResult.getAccumulatorResults(),
-					classLoader));
-			return lastJobExecutionResult;
-		} catch (IOException | ClassNotFoundException e) {
-			throw new ProgramInvocationException(e);
+			final JobResult jobResult;
+			try {
+				jobResult = jobResultFuture.get();
+			} catch (Exception e) {
+				throw new ProgramInvocationException("Could not retrieve the execution result.", ExceptionUtils.stripExecutionException(e));
+			}
+
+			if (jobResult.getSerializedThrowable().isPresent()) {
+				final SerializedThrowable serializedThrowable = jobResult.getSerializedThrowable().get();
+				final Throwable throwable = serializedThrowable.deserializeError(classLoader);
+				throw new ProgramInvocationException(throwable);
+			}
+
+			try {
+				this.lastJobExecutionResult = new JobExecutionResult(
+					jobResult.getJobId(),
+					jobResult.getNetRuntime(),
+					AccumulatorHelper.deserializeAccumulators(
+						jobResult.getAccumulatorResults(),
+						classLoader));
+				return lastJobExecutionResult;
+			} catch (IOException | ClassNotFoundException e) {
+				throw new ProgramInvocationException(e);
+			}
 		}
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/f57b9833/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java
index c880817..9e75eea 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java
@@ -20,6 +20,7 @@ package org.apache.flink.client.program.rest;
 
 import org.apache.flink.api.common.JobExecutionResult;
 import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobSubmissionResult;
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.client.deployment.StandaloneClusterId;
@@ -115,6 +116,8 @@ import static org.apache.flink.util.Preconditions.checkArgument;
 import static org.apache.flink.util.Preconditions.checkState;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
@@ -147,6 +150,9 @@ public class RestClusterClientTest extends TestLogger {
 
 	private ExecutorService executor;
 
+	private JobGraph jobGraph;
+	private JobID jobId;
+
 	@Before
 	public void setUp() throws Exception {
 		MockitoAnnotations.initMocks(this);
@@ -178,6 +184,9 @@ public class RestClusterClientTest extends TestLogger {
 			}
 		};
 		restClusterClient = new RestClusterClient<>(config, restClient, StandaloneClusterId.getInstance(), (attempt) -> 0);
+
+		jobGraph = new JobGraph("testjob");
+		jobId = jobGraph.getJobID();
 	}
 
 	@After
@@ -193,16 +202,13 @@ public class RestClusterClientTest extends TestLogger {
 
 	@Test
 	public void testJobSubmitCancelStop() throws Exception {
-		final JobGraph job = new JobGraph("testjob");
-		final JobID id = job.getJobID();
-
 		TestBlobServerPortHandler portHandler = new TestBlobServerPortHandler();
 		TestJobSubmitHandler submitHandler = new TestJobSubmitHandler();
 		TestJobTerminationHandler terminationHandler = new TestJobTerminationHandler();
 		TestJobExecutionResultHandler testJobExecutionResultHandler =
 			new TestJobExecutionResultHandler(
 				JobExecutionResultResponseBody.created(new JobResult.Builder()
-					.jobId(id)
+					.jobId(jobId)
 					.netRuntime(Long.MAX_VALUE)
 					.build()));
 
@@ -214,20 +220,44 @@ public class RestClusterClientTest extends TestLogger {
 
 			Assert.assertFalse(portHandler.portRetrieved);
 			Assert.assertFalse(submitHandler.jobSubmitted);
-			restClusterClient.submitJob(job, ClassLoader.getSystemClassLoader());
+			restClusterClient.submitJob(jobGraph, ClassLoader.getSystemClassLoader());
 			Assert.assertTrue(portHandler.portRetrieved);
 			Assert.assertTrue(submitHandler.jobSubmitted);
 
 			Assert.assertFalse(terminationHandler.jobCanceled);
-			restClusterClient.cancel(id);
+			restClusterClient.cancel(jobId);
 			Assert.assertTrue(terminationHandler.jobCanceled);
 
 			Assert.assertFalse(terminationHandler.jobStopped);
-			restClusterClient.stop(id);
+			restClusterClient.stop(jobId);
 			Assert.assertTrue(terminationHandler.jobStopped);
 		}
 	}
 
+	/**
+	 * Tests that we can submit a jobGraph in detached mode.
+	 */
+	@Test
+	public void testDetachedJobSubmission() throws Exception {
+
+		final TestBlobServerPortHandler testBlobServerPortHandler = new TestBlobServerPortHandler();
+		final TestJobSubmitHandler testJobSubmitHandler = new TestJobSubmitHandler();
+
+		try (TestRestServerEndpoint ignored = createRestServerEndpoint(
+			testBlobServerPortHandler,
+			testJobSubmitHandler)) {
+
+			restClusterClient.setDetached(true);
+			final JobSubmissionResult jobSubmissionResult = restClusterClient.submitJob(jobGraph, ClassLoader.getSystemClassLoader());
+
+			// if the detached mode didn't work, then we would not reach this point because the execution result
+			// retrieval would have failed.
+			assertThat(jobSubmissionResult, is(not(instanceOf(JobExecutionResult.class))));
+			assertThat(jobSubmissionResult.getJobID(), is(jobId));
+		}
+
+	}
+
 	private class TestBlobServerPortHandler extends TestHandler<EmptyRequestBody, BlobServerPortResponseBody, EmptyMessageParameters> {
 		private volatile boolean portRetrieved = false;
 
@@ -314,9 +344,6 @@ public class RestClusterClientTest extends TestLogger {
 
 	@Test
 	public void testSubmitJobAndWaitForExecutionResult() throws Exception {
-		final JobGraph jobGraph = new JobGraph("testjob");
-		final JobID jobId = jobGraph.getJobID();
-
 		final TestJobExecutionResultHandler testJobExecutionResultHandler =
 			new TestJobExecutionResultHandler(
 				new RestHandlerException("should trigger retry", HttpResponseStatus.NOT_FOUND),
@@ -509,7 +536,7 @@ public class RestClusterClientTest extends TestLogger {
 				Iterator<JobStatusMessage> jobDetailsIterator = jobDetails.iterator();
 				JobStatusMessage job1 = jobDetailsIterator.next();
 				JobStatusMessage job2 = jobDetailsIterator.next();
-				Assert.assertNotEquals("The job statues should not be equal.", job1.getJobState(), job2.getJobState());
+				Assert.assertNotEquals("The job status should not be equal.", job1.getJobState(), job2.getJobState());
 			}
 		}
 	}


[18/22] flink git commit: [FLINK-8644] [flip6] Shut down AkkaRpcActors with PoisonPill

Posted by tr...@apache.org.
[FLINK-8644] [flip6] Shut down AkkaRpcActors with PoisonPill

Shutting AkkaRpcActors down with Kill can result in uncompleted futures, if
the mailbox contains requests which will be dropped. Therefore, it is better
to stop AkkaRpcActors with a PoisonPill which assures that all messages before
the pill will be processed.

This closes #5476.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5ed180c8
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5ed180c8
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5ed180c8

Branch: refs/heads/master
Commit: 5ed180c8121992d1b64dc879c9b6d78d00e57c92
Parents: f57b983
Author: Till Rohrmann <tr...@apache.org>
Authored: Tue Feb 13 14:27:31 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Thu Feb 15 08:45:46 2018 +0100

----------------------------------------------------------------------
 .../java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/5ed180c8/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
index 7ff08f7..d2d4bf2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
@@ -40,7 +40,7 @@ import akka.actor.ActorSelection;
 import akka.actor.ActorSystem;
 import akka.actor.Address;
 import akka.actor.Identify;
-import akka.actor.Kill;
+import akka.actor.PoisonPill;
 import akka.actor.Props;
 import akka.dispatch.Futures;
 import akka.dispatch.Mapper;
@@ -303,7 +303,7 @@ public class AkkaRpcService implements RpcService {
 			}
 
 			if (rpcEndpoint != null) {
-				akkaClient.getActorRef().tell(Kill.getInstance(), ActorRef.noSender());
+				akkaClient.getActorRef().tell(PoisonPill.getInstance(), ActorRef.noSender());
 			} else {
 				LOG.debug("RPC endpoint {} already stopped or from different RPC service", selfGateway.getAddress());
 			}


[05/22] flink git commit: [FLINK-8605] [rest] Enable job cancellation from the web UI

Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime-web/web-dashboard/web/js/hs/index.js
----------------------------------------------------------------------
diff --git a/flink-runtime-web/web-dashboard/web/js/hs/index.js b/flink-runtime-web/web-dashboard/web/js/hs/index.js
index 72e1d26..7cd45e1 100644
--- a/flink-runtime-web/web-dashboard/web/js/hs/index.js
+++ b/flink-runtime-web/web-dashboard/web/js/hs/index.js
@@ -1,2 +1,2 @@
-angular.module("flinkApp",["ui.router","angularMoment","dndLists"]).run(["$rootScope",function(e){return e.sidebarVisible=!1,e.showSidebar=function(){return e.sidebarVisible=!e.sidebarVisible,e.sidebarClass="force-show"}}]).value("flinkConfig",{jobServer:"","refresh-interval":1e4}).value("watermarksConfig",{noWatermark:-0x8000000000000000}).run(["JobsService","MainService","flinkConfig","$interval",function(e,t,r,n){return t.loadConfig().then(function(t){return angular.extend(r,t),e.listJobs(),n(function(){return e.listJobs()},r["refresh-interval"])})}]).config(["$uiViewScrollProvider",function(e){return e.useAnchorScroll()}]).run(["$rootScope","$state",function(e,t){return e.$on("$stateChangeStart",function(e,r,n,i){if(r.redirectTo)return e.preventDefault(),t.go(r.redirectTo,n)})}]).config(["$stateProvider","$urlRouterProvider",function(e,t){return e.state("completed-jobs",{url:"/completed-jobs",views:{main:{templateUrl:"partials/jobs/completed-jobs.html",controller:"CompletedJobsC
 ontroller"}}}).state("single-job",{url:"/jobs/{jobid}","abstract":!0,views:{main:{templateUrl:"partials/jobs/job.html",controller:"SingleJobController"}}}).state("single-job.plan",{url:"",redirectTo:"single-job.plan.subtasks",views:{details:{templateUrl:"partials/jobs/job.plan.html",controller:"JobPlanController"}}}).state("single-job.plan.subtasks",{url:"",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.subtasks.html",controller:"JobPlanSubtasksController"}}}).state("single-job.plan.metrics",{url:"/metrics",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.metrics.html",controller:"JobPlanMetricsController"}}}).state("single-job.plan.watermarks",{url:"/watermarks",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.watermarks.html"}}}).state("single-job.plan.taskmanagers",{url:"/taskmanagers",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.taskmanagers.html",controller:"JobPlanTaskManagersController"}}}).
 state("single-job.plan.accumulators",{url:"/accumulators",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.accumulators.html",controller:"JobPlanAccumulatorsController"}}}).state("single-job.plan.checkpoints",{url:"/checkpoints",redirectTo:"single-job.plan.checkpoints.overview",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.checkpoints.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.overview",{url:"/overview",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.overview.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.summary",{url:"/summary",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.summary.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.history",{url:"/history",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.history.html",controll
 er:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.config",{url:"/config",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.config.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.details",{url:"/details/{checkpointId}",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.details.html",controller:"JobPlanCheckpointDetailsController"}}}).state("single-job.plan.backpressure",{url:"/backpressure",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.backpressure.html",controller:"JobPlanBackPressureController"}}}).state("single-job.timeline",{url:"/timeline",views:{details:{templateUrl:"partials/jobs/job.timeline.html"}}}).state("single-job.timeline.vertex",{url:"/{vertexId}",views:{vertex:{templateUrl:"partials/jobs/job.timeline.vertex.html",controller:"JobTimelineVertexController"}}}).state("single-job.exceptions",{url:"/exceptions",views:{details:{templ
 ateUrl:"partials/jobs/job.exceptions.html",controller:"JobExceptionsController"}}}).state("single-job.config",{url:"/config",views:{details:{templateUrl:"partials/jobs/job.config.html"}}}),t.otherwise("/completed-jobs")}]),angular.module("flinkApp").directive("bsLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getLabelClass=function(){return"label label-"+e.translateLabelState(n.status)}}}}]).directive("bpLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getBackPressureLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getBackPressureLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getBackPressureLabelClass=function(){return"label label-"+e.translateBackPressureLabelState(n.status)}}}}]).directive("indicatorPrimary",["Jobs
 Service",function(e){return{replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<i title='{{status}}' ng-class='getLabelClass()' />",link:function(t,r,n){return t.getLabelClass=function(){return"fa fa-circle indicator indicator-"+e.translateLabelState(n.status)}}}}]).directive("tableProperty",function(){return{replace:!0,scope:{value:"="},template:"<td title=\"{{value || 'None'}}\">{{value || 'None'}}</td>"}}),angular.module("flinkApp").filter("amDurationFormatExtended",["angularMomentConfig",function(e){var t;return t=function(e,t,r){return"undefined"==typeof e||null===e?"":moment.duration(e,t).format(r,{trim:!1})},t.$stateful=e.statefulFilters,t}]).filter("humanizeDuration",function(){return function(e,t){var r,n,i,o,s,a;return"undefined"==typeof e||null===e?"":(o=e%1e3,a=Math.floor(e/1e3),s=a%60,a=Math.floor(a/60),i=a%60,a=Math.floor(a/60),n=a%24,a=Math.floor(a/24),r=a,0===r?0===n?0===i?0===s?o+"ms":s+"s ":i+"m "+s+"s":t?n+"h "+i+"m":n+"h "+i+"m "+s+"s":t?r+"d "+n+"h":r+"d 
 "+n+"h "+i+"m "+s+"s")}}).filter("limit",function(){return function(e){return e.length>73&&(e=e.substring(0,35)+"..."+e.substring(e.length-35,e.length)),e}}).filter("humanizeText",function(){return function(e){return e?e.replace(/&gt;/g,">").replace(/<br\/>/g,""):""}}).filter("humanizeBytes",function(){return function(e){var t,r;return r=["B","KB","MB","GB","TB","PB","EB"],t=function(e,n){var i;return i=Math.pow(1024,n),e<i?(e/i).toFixed(2)+" "+r[n]:e<1e3*i?(e/i).toPrecision(3)+" "+r[n]:t(e,n+1)},"undefined"==typeof e||null===e?"":e<1e3?e+" B":t(e,1)}}).filter("toLocaleString",function(){return function(e){return e.toLocaleString()}}).filter("toUpperCase",function(){return function(e){return e.toUpperCase()}}).filter("percentage",function(){return function(e){return(100*e).toFixed(0)+"%"}}).filter("humanizeWatermark",["watermarksConfig",function(e){return function(t){return isNaN(t)||t<=e.noWatermark?"No Watermark":t}}]).filter("increment",function(){return function(e){return parseI
 nt(e)+1}}).filter("humanizeChartNumeric",["humanizeBytesFilter","humanizeDurationFilter",function(e,t){return function(r,n){var i;return i="",null!==r&&(i=/bytes/i.test(n.id)&&/persecond/i.test(n.id)?e(r)+" / s":/bytes/i.test(n.id)?e(r):/persecond/i.test(n.id)?r+" / s":/time/i.test(n.id)||/latency/i.test(n.id)?t(r,!0):r),i}}]).filter("humanizeChartNumericTitle",["humanizeDurationFilter",function(e){return function(t,r){var n;return n="",null!==t&&(n=/bytes/i.test(r.id)&&/persecond/i.test(r.id)?t+" Bytes / s":/bytes/i.test(r.id)?t+" Bytes":/persecond/i.test(r.id)?t+" / s":/time/i.test(r.id)||/latency/i.test(r.id)?e(t,!1):t),n}}]).filter("searchMetrics",function(){return function(e,t){var r,n;return n=new RegExp(t,"gi"),function(){var t,i,o;for(o=[],t=0,i=e.length;t<i;t++)r=e[t],r.id.match(n)&&o.push(r);return o}()}}),angular.module("flinkApp").service("MainService",["$http","flinkConfig","$q",function(e,t,r){return this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer
 +"config").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this}]),angular.module("flinkApp").controller("RunningJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("running")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("CompletedJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("finished")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("SingleJobController",["$scope","$state","$stateParams","JobsService","MetricsService","$rootScope","flinkConfig","$interval","$q","watermarksConfig",function(e,t,r,n,i,o,s,a,l,u){var c,d;return e.jobid=r.jobid,e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperator
 Stats={},d=a(function(){return n.loadJob(r.jobid).then(function(t){return e.job=t,e.$broadcast("reload")})},s["refresh-interval"]),e.$on("$destroy",function(){return e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats=null,a.cancel(d)}),e.cancelJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Cancelling..."),n.cancelJob(r.jobid).then(function(e){return{}})},e.stopJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Stopping..."),n.stopJob(r.jobid).then(function(e){return{}})},n.loadJob(r.jobid).then(function(t){return e.job=t,e.vertices=t.vertices,e.plan=t.plan,i.setupMetrics(r.jobid,t.vertices)}),c=function(t){var r,n,o,s;return o=function(t){return function(t){var r,n,o,s;return r=l.defer(),o=e.job.jid,s=function(){var e,r,i;for(i=[],n=e=0,r=t.parallelism-1;0<=r?e<=r:e>=r;n=0<=r?++e:--e)i.push(n+".currentLowWatermark");return i}(),i.getMe
 trics(o,t.id,s).then(function(e){var t,n,i,o,s,a,l;i=NaN,l={},o=e.values;for(t in o)a=o[t],s=t.replace(".currentLowWatermark",""),l[s]=a,(isNaN(i)||a<i)&&(i=a);return n=!isNaN(i)&&i>u.noWatermark?i:NaN,r.resolve({lowWatermark:n,watermarks:l})}),r.promise}}(this),r=l.defer(),s={},n=t.length,angular.forEach(t,function(e){return function(e,t){var i;return i=e.id,o(e).then(function(e){if(s[i]=e,t>=n-1)return r.resolve(s)})}}(this)),r.promise},e.hasWatermark=function(t){return e.watermarks[t]&&!isNaN(e.watermarks[t].lowWatermark)},e.$watch("plan",function(t){if(t)return c(t.nodes).then(function(t){return e.watermarks=t})}),e.$on("reload",function(){if(e.plan)return c(e.plan.nodes).then(function(t){return e.watermarks=t})})}]).controller("JobPlanController",["$scope","$state","$stateParams","$window","JobsService",function(e,t,r,n,i){return e.nodeid=null,e.nodeUnfolded=!1,e.stateList=i.stateList(),e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.acc
 umulators=null,e.operatorCheckpointStats=null,e.$broadcast("reload"),e.$broadcast("node:change",e.nodeid)):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null)},e.deactivateNode=function(){return e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null},e.toggleFold=function(){return e.nodeUnfolded=!e.nodeUnfolded}}]).controller("JobPlanSubtasksController",["$scope","JobsService",function(e,t){var r;return e.aggregate=!1,r=function(){return e.aggregate?t.getTaskManagers(e.nodeid).then(function(t){return e.taskmanagers=t}):t.getSubtasks(e.nodeid).then(function(t){return e.subtasks=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanAccumulatorsController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getAccumulators(e.nodeid).then(function(t){return e.accumulators=t.main,e.subt
 askAccumulators=t.subtasks})},!e.nodeid||e.vertex&&e.vertex.accumulators||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanCheckpointsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return e.checkpointDetails={},e.checkpointDetails.id=-1,n.getCheckpointConfig().then(function(t){return e.checkpointConfig=t}),i=function(){return n.getCheckpointStats().then(function(t){if(null!==t)return e.checkpointStats=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobPlanCheckpointDetailsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i,o;return e.subtaskDetails={},e.checkpointDetails.id=r.checkpointId,i=function(t){return n.getCheckpointDetails(t).then(function(t){return null!==t?e.checkpoint=t:e.unknown_checkpoint=!0})},o=function(t,r){return n.getCheckpointSubtaskDetails(t,r).then(function(t){if(null!==t)return e.subtaskDetails[r]=t})},i(r.checkpointId),e.nodeid&&o(r.checkpointId,e
 .nodeid),e.$on("reload",function(t){if(i(r.checkpointId),e.nodeid)return o(r.checkpointId,e.nodeid)}),e.$on("$destroy",function(){return e.checkpointDetails.id=-1})}]).controller("JobPlanBackPressureController",["$scope","JobsService",function(e,t){var r;return r=function(){if(e.now=Date.now(),e.nodeid)return t.getOperatorBackPressure(e.nodeid).then(function(t){return e.backPressureOperatorStats[e.nodeid]=t})},r(),e.$on("reload",function(e){return r()})}]).controller("JobTimelineVertexController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return i=function(){return n.getVertex(r.vertexId).then(function(t){return e.vertex=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobExceptionsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return n.loadExceptions().then(function(t){return e.exceptions=t})}]).controller("JobPropertiesController",["$scope","JobsService",function(e,t){return e.changeNode=function(r){return 
 r!==e.nodeid?(e.nodeid=r,t.getNode(r).then(function(t){return e.node=t})):(e.nodeid=null,e.node=null)}}]).controller("JobPlanMetricsController",["$scope","JobsService","MetricsService",function(e,t,r){var n,i;if(e.dragging=!1,e.window=r.getWindow(),e.availableMetrics=null,e.$on("$destroy",function(){return r.unRegisterObserver()}),i=function(){return t.getVertex(e.nodeid).then(function(t){return e.vertex=t}),r.getAvailableMetrics(e.jobid,e.nodeid).then(function(t){return e.availableMetrics=t.sort(n),e.metrics=r.getMetricsSetup(e.jobid,e.nodeid).names,r.registerObserver(e.jobid,e.nodeid,function(t){return e.$broadcast("metrics:data:update",t.timestamp,t.values)})})},n=function(e,t){var r,n;return r=e.id.toLowerCase(),n=t.id.toLowerCase(),r<n?-1:r>n?1:0},e.dropped=function(t,n,o,s,a){return r.orderMetrics(e.jobid,e.nodeid,o,n),e.$broadcast("metrics:refresh",o),i(),!1},e.dragStart=function(){return e.dragging=!0},e.dragEnd=function(){return e.dragging=!1},e.addMetric=function(t){return
  r.addMetric(e.jobid,e.nodeid,t.id),i()},e.removeMetric=function(t){return r.removeMetric(e.jobid,e.nodeid,t),i()},e.setMetricSize=function(t,n){return r.setMetricSize(e.jobid,e.nodeid,t,n),i()},e.setMetricView=function(t,n){return r.setMetricView(e.jobid,e.nodeid,t,n),i()},e.getValues=function(t){return r.getValues(e.jobid,e.nodeid,t)},e.$on("node:change",function(t,r){if(!e.dragging)return i()}),e.nodeid)return i()}]),angular.module("flinkApp").directive("vertex",["$state",function(e){return{template:"<svg class='timeline secondary' width='0' height='0'></svg>",scope:{data:"="},link:function(e,t,r){var n,i,o;o=t.children()[0],i=t.width(),angular.element(o).attr("width",i),(n=function(e){var t,r,n;return d3.select(o).selectAll("*").remove(),n=[],angular.forEach(e.subtasks,function(e,t){var r;return r=[{label:"Scheduled",color:"#666",borderColor:"#555",starting_time:e.timestamps.SCHEDULED,ending_time:e.timestamps.DEPLOYING,type:"regular"},{label:"Deploying",color:"#aaa",borderColor:
 "#555",starting_time:e.timestamps.DEPLOYING,ending_time:e.timestamps.RUNNING,type:"regular"}],e.timestamps.FINISHED>0&&r.push({label:"Running",color:"#ddd",borderColor:"#555",starting_time:e.timestamps.RUNNING,ending_time:e.timestamps.FINISHED,type:"regular"}),n.push({label:"("+e.subtask+") "+e.host,times:r})}),t=d3.timeline().stack().tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("single").labelFormat(function(e){return e}).margin({left:100,right:0,top:0,bottom:0}).itemHeight(30).relativeTime(),r=d3.select(o).datum(n).call(t)})(e.data)}}}]).directive("timeline",["$state",function(e){return{template:"<svg class='timeline' width='0' height='0'></svg>",scope:{vertices:"=",jobid:"="},link:function(t,r,n){var i,o,s,a;s=r.children()[0],o=r.width(),angular.element(s).attr("width",o),a=function(e){return e.replace("&gt;",">")},i=function(r){var n,i,o;return d3.select(s).selectAll("*").remove(),o=[],angular.forEach(r,function(e){if(e["start-time"]>-1)return"scheduled"===e.type?
 o.push({times:[{label:a(e.name),color:"#cccccc",borderColor:"#555555",starting_time:e["start-time"],ending_time:e["end-time"],type:e.type}]}):o.push({times:[{label:a(e.name),color:"#d9f1f7",borderColor:"#62cdea",starting_time:e["start-time"],ending_time:e["end-time"],link:e.id,type:e.type}]})}),n=d3.timeline().stack().click(function(r,n,i){if(r.link)return e.go("single-job.timeline.vertex",{jobid:t.jobid,vertexId:r.link})}).tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("main").margin({left:0,right:0,top:0,bottom:0}).itemHeight(30).showBorderLine().showHourTimeline(),i=d3.select(s).datum(o).call(n)},t.$watch(n.vertices,function(e){if(e)return i(e)})}}}]).directive("split",function(){return{compile:function(e,t){return Split(e.children(),{sizes:[50,50],direction:"vertical"})}}}).directive("jobPlan",["$timeout",function(e){return{template:"<svg class='graph'><g /></svg> <svg class='tmp' width='1' height='1'><g /></svg> <div class='btn-group zoom-buttons'> <a class='btn bt
 n-default zoom-in' ng-click='zoomIn()'><i class='fa fa-plus' /></a> <a class='btn btn-default zoom-out' ng-click='zoomOut()'><i class='fa fa-minus' /></a> </div>",scope:{plan:"=",watermarks:"=",setNode:"&"},link:function(e,t,r){var n,i,o,s,a,l,u,c,d,f,p,m,h,g,b,v,k,j,S,w,C,$,y,J,M;p=null,C=d3.behavior.zoom(),M=[],g=r.jobid,S=t.children()[0],j=t.children().children()[0],w=t.children()[1],l=d3.select(S),u=d3.select(j),c=d3.select(w),n=t.width(),angular.element(t.children()[0]).width(n),v=0,b=0,e.zoomIn=function(){var e,t,r;if(C.scale()<2.99)return e=C.translate(),t=e[0]*(C.scale()+.1/C.scale()),r=e[1]*(C.scale()+.1/C.scale()),C.scale(C.scale()+.1),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},e.zoomOut=function(){var e,t,r;if(C.scale()>.31)return C.scale(C.scale()-.1),e=C.translate(),t=e[0]*(C.scale()-.1/C.scale()),r=e[1]*(C.scale()-.1/C.scale()),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.
 scale()+")"),v=C.scale(),b=C.translate()},o=function(e){var t;return t="",null==e.ship_strategy&&null==e.local_strategy||(t+="<div class='edge-label'>",null!=e.ship_strategy&&(t+=e.ship_strategy),void 0!==e.temp_mode&&(t+=" ("+e.temp_mode+")"),void 0!==e.local_strategy&&(t+=",<br>"+e.local_strategy),t+="</div>"),t},h=function(e){return"partialSolution"===e||"nextPartialSolution"===e||"workset"===e||"nextWorkset"===e||"solutionSet"===e||"solutionDelta"===e},m=function(e,t){return"mirror"===t?"node-mirror":h(t)?"node-iteration":"node-normal"},s=function(e,t,r,n){var i,o;return i="<div href='#/jobs/"+g+"/vertex/"+e.id+"' class='node-label "+m(e,t)+"'>",i+="mirror"===t?"<h3 class='node-name'>Mirror of "+e.operator+"</h3>":"<h3 class='node-name'>"+e.operator+"</h3>",""===e.description?i+="":(o=e.description,o=J(o),i+="<h4 class='step-name'>"+o+"</h4>"),null!=e.step_function?i+=f(e.id,r,n):(h(t)&&(i+="<h5>"+t+" Node</h5>"),""!==e.parallelism&&(i+="<h5>Parallelism: "+e.parallelism+"</h5>")
 ,void 0!==e.lowWatermark&&(i+="<h5>Low Watermark: "+e.lowWatermark+"</h5>"),void 0!==e.operator&&e.operator_strategy&&(i+="<h5>Operation: "+J(e.operator_strategy)+"</h5>")),i+="</div>"},f=function(e,t,r){var n,i;return i="svg-"+e,n="<svg class='"+i+"' width="+t+" height="+r+"><g /></svg>"},J=function(e){var t;for("<"===e.charAt(0)&&(e=e.replace("<","&lt;"),e=e.replace(">","&gt;")),t="";e.length>30;)t=t+e.substring(0,30)+"<br>",e=e.substring(30,e.length);return t+=e},a=function(e,t,r,n,i,o){return null==n&&(n=!1),r.id===t.partial_solution?e.setNode(r.id,{label:s(r,"partialSolution",i,o),labelType:"html","class":m(r,"partialSolution")}):r.id===t.next_partial_solution?e.setNode(r.id,{label:s(r,"nextPartialSolution",i,o),labelType:"html","class":m(r,"nextPartialSolution")}):r.id===t.workset?e.setNode(r.id,{label:s(r,"workset",i,o),labelType:"html","class":m(r,"workset")}):r.id===t.next_workset?e.setNode(r.id,{label:s(r,"nextWorkset",i,o),labelType:"html","class":m(r,"nextWorkset")}):r.i
 d===t.solution_set?e.setNode(r.id,{label:s(r,"solutionSet",i,o),labelType:"html","class":m(r,"solutionSet")}):r.id===t.solution_delta?e.setNode(r.id,{label:s(r,"solutionDelta",i,o),labelType:"html","class":m(r,"solutionDelta")}):e.setNode(r.id,{label:s(r,"",i,o),labelType:"html","class":m(r,"")})},i=function(e,t,r,n,i){return e.setEdge(i.id,r.id,{label:o(i),labelType:"html",arrowhead:"normal"})},k=function(e,t){var r,n,o,s,l,u,d,f,p,m,h,g,b,v;for(n=[],null!=t.nodes?v=t.nodes:(v=t.step_function,o=!0),s=0,u=v.length;s<u;s++)if(r=v[s],p=0,f=0,r.step_function&&(b=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:20,edgesep:0,ranksep:20,rankdir:"LR",marginx:10,marginy:10}),M[r.id]=b,k(b,r),h=new dagreD3.render,c.select("g").call(h,b),p=b.graph().width,f=b.graph().height,angular.element(w).empty()),a(e,t,r,o,p,f),n.push(r.id),null!=r.inputs)for(g=r.inputs,l=0,d=g.length;l<d;l++)m=g[l],i(e,t,r,n,m);return e},y=function(e,t){var r,n,i;for(n in e.nodes){if(r=e.nodes[n
 ],r.id===t)return r;if(null!=r.step_function)for(i in r.step_function)if(r.step_function[i].id===t)return r.step_function[i]}},$=function(e,t){var r,n,i,o;if(!_.isEmpty(t))for(o=e.nodes,r=0,n=o.length;r<n;r++)i=o[r],t[i.id]&&!isNaN(t[i.id].lowWatermark)&&(i.lowWatermark=t[i.id].lowWatermark);return e},b=0,v=0,d=function(){var t,r,n,i,o,s;if(e.plan){p=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:70,edgesep:0,ranksep:50,rankdir:"LR",marginx:40,marginy:40}),k(p,$(e.plan,e.watermarks)),u.selectAll("*").remove(),u.attr("transform","scale(1)"),n=new dagreD3.render,u.call(n,p);for(t in M)i=M[t],l.select("svg.svg-"+t+" g").call(n,i);return r=.5,o=Math.floor((angular.element(S).width()-p.graph().width*r)/2),s=Math.floor((angular.element(S).height()-p.graph().height*r)/2),0!==v&&0!==b?(C.scale(v).translate(b),u.attr("transform","translate("+b+") scale("+v+")")):(C.scale(r).translate([o,s]),u.attr("transform","translate("+o+", "+s+") scale("+C.scale()+")")),C.on("z
 oom",function(){var e;return e=d3.event,v=e.scale,b=e.translate,u.attr("transform","translate("+b+") scale("+v+")")}),C(l),u.selectAll(".node").on("click",function(t){return e.setNode({nodeid:t})})}},e.$watch(r.plan,function(e){if(e)return d()}),e.$watch(r.watermarks,function(t){if(t&&e.plan)return d()})}}}]),angular.module("flinkApp").service("JobsService",["$http","flinkConfig","$log","amMoment","$q","$timeout",function(e,t,r,n,i,o){var s,a,l,u,c,d;return s=null,a=null,l={},c={running:[],finished:[],cancelled:[],failed:[]},u=[],d=function(){return angular.forEach(u,function(e){return e()})},this.registerObserver=function(e){return u.push(e)},this.unRegisterObserver=function(e){var t;return t=u.indexOf(e),u.splice(t,1)},this.stateList=function(){return["SCHEDULED","DEPLOYING","RUNNING","FINISHED","FAILED","CANCELING","CANCELED"]},this.translateLabelState=function(e){switch(e.toLowerCase()){case"finished":return"success";case"failed":return"danger";case"scheduled":return"default";ca
 se"deploying":return"info";case"running":return"primary";case"canceling":return"warning";case"pending":return"info";case"total":return"black";default:return"default"}},this.setEndTimes=function(e){return angular.forEach(e,function(e,t){if(!(e["end-time"]>-1))return e["end-time"]=e["start-time"]+e.duration})},this.processVertices=function(e){return angular.forEach(e.vertices,function(e,t){return e.type="regular"}),e.vertices.unshift({name:"Scheduled","start-time":e.timestamps.CREATED,"end-time":e.timestamps.CREATED+1,type:"scheduled"})},this.listJobs=function(){var r;return r=i.defer(),e.get(t.jobServer+"jobs/overview").success(function(e){return function(t,n,i,o){return c.finished=[],c.running=[],_(t.jobs).groupBy(function(e){switch(e.state.toLowerCase()){case"finished":return"finished";case"failed":return"finished";case"canceled":return"finished";default:return"running"}}).forEach(function(t,r){switch(r){case"finished":return c.finished=e.setEndTimes(t);case"running":return c.runni
 ng=e.setEndTimes(t)}}).value(),r.resolve(c),d()}}(this)),r.promise},this.getJobs=function(e){return c[e]},this.getAllJobs=function(){return c},this.loadJob=function(r){return s=null,l.job=i.defer(),e.get(t.jobServer+"jobs/"+r).success(function(n){return function(i,o,a,u){return n.setEndTimes(i.vertices),n.processVertices(i),e.get(t.jobServer+"jobs/"+r+"/config").success(function(e){return i=angular.extend(i,e),s=i,l.job.resolve(s)})}}(this)),l.job.promise},this.getNode=function(e){var t,r;return r=function(e,t){var n,i,o,s;for(n=0,i=t.length;n<i;n++){if(o=t[n],o.id===e)return o;if(o.step_function&&(s=r(e,o.step_function)),s)return s}return null},t=i.defer(),l.job.promise.then(function(n){return function(i){var o;return o=r(e,s.plan.nodes),o.vertex=n.seekVertex(e),t.resolve(o)}}(this)),t.promise},this.seekVertex=function(e){var t,r,n,i;for(n=s.vertices,t=0,r=n.length;t<r;t++)if(i=n[t],i.id===e)return i;return null},this.getVertex=function(r){var n;return n=i.defer(),l.job.promise.the
 n(function(i){return function(o){var a;return a=i.seekVertex(r),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/subtasktimes").success(function(e){return a.subtasks=e.subtasks,n.resolve(a)})}}(this)),n.promise},this.getSubtasks=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r).success(function(e){var t;return t=e.subtasks,n.resolve(t)})}}(this)),n.promise},this.getTaskManagers=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/taskmanagers").success(function(e){var t;return t=e.taskmanagers,n.resolve(t)})}}(this)),n.promise},this.getAccumulators=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return console.log(s.jid),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/accumulators").success(function(i){var o;return o=i["user-accumulators"],e.get(t.jobServer+"jobs/"
 +s.jid+"/vertices/"+r+"/subtasks/accumulators").success(function(e){var t;return t=e.subtasks,n.resolve({main:o,subtasks:t})})})}}(this)),n.promise},this.getCheckpointConfig=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/config").success(function(e){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointStats=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints").success(function(e,t,n,i){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointDetails=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/details/"+r).success(function(e){return angular.equals({},e)?n.resolve(null):n.resolve(e)})}}(this)),n.promise},this.getCheckpointSubtaskDet
 ails=function(r,n){var o;return o=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/details/"+r+"/subtasks/"+n).success(function(e){return angular.equals({},e)?o.resolve(null):o.resolve(e)})}}(this)),o.promise},this.getOperatorBackPressure=function(r){var n;return n=i.defer(),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/backpressure").success(function(e){return function(e){return n.resolve(e)}}(this)),n.promise},this.translateBackPressureLabelState=function(e){switch(e.toLowerCase()){case"in-progress":return"danger";case"ok":return"success";case"low":return"warning";case"high":return"danger";default:return"default"}},this.loadExceptions=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/exceptions").success(function(e){return s.exceptions=e,r.resolve(e)})}}(this)),r.promise},this.cancelJob=function(r){return e.get(t.jobServer+"jobs/"+r+"/yarn
 -cancel")},this.stopJob=function(t){return e.get("jobs/"+t+"/yarn-stop")},this}]),angular.module("flinkApp").directive("metricsGraph",function(){return{template:'<div class="panel panel-default panel-metric"> <div class="panel-heading"> <span class="metric-title">{{metric.id}}</span> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.size != \'big\'}]" ng-click="setSize(\'small\')">Small</button> <button type="button" ng-class="[btnClasses, {active: metric.size == \'big\'}]" ng-click="setSize(\'big\')">Big</button> </div> <a title="Remove" class="btn btn-default btn-xs remove" ng-click="removeMetric()"><i class="fa fa-close" /></a> </div> </div> <div class="panel-body"> <svg ng-if="metric.view == \'chart\'"/> <div ng-if="metric.view != \'chart\'"> <div class="metric-numeric" title="{{value | humanizeChartNumericTitle:metric}}">{{value | humanizeChartNumeric:metric}}</div> </div> </div> <div class="buttons"> <div class="btn-grou
 p"> <button type="button" ng-class="[btnClasses, {active: metric.view == \'chart\'}]" ng-click="setView(\'chart\')">Chart</button> <button type="button" ng-class="[btnClasses, {active: metric.view != \'chart\'}]" ng-click="setView(\'numeric\')">Numeric</button> </div> </div>',replace:!0,scope:{metric:"=",window:"=",removeMetric:"&",setMetricSize:"=",setMetricView:"=",getValues:"&"},link:function(e,t,r){return e.btnClasses=["btn","btn-default","btn-xs"],e.value=null,e.data=[{values:e.getValues()}],e.options={x:function(e,t){return e.x},y:function(e,t){return e.y},xTickFormat:function(e){return d3.time.format("%H:%M:%S")(new Date(e))},yTickFormat:function(e){var t,r,n,i;for(r=!1,n=0,i=1,t=Math.abs(e);!r&&n<50;)Math.pow(10,n)<=t&&t<Math.pow(10,n+i)?r=!0:n+=i;return r&&n>6?e/Math.pow(10,n)+"E"+n:""+e}},e.showChart=function(){return d3.select(t.find("svg")[0]).datum(e.data).transition().duration(250).call(e.chart)},e.chart=nv.models.lineChart().options(e.options).showLegend(!1).margin({t
 op:15,left:60,bottom:30,right:30}),e.chart.yAxis.showMaxMin(!1),e.chart.tooltip.hideDelay(0),e.chart.tooltip.contentGenerator(function(e){return"<p>"+d3.time.format("%H:%M:%S")(new Date(e.point.x))+" | "+e.point.y+"</p>"}),nv.utils.windowResize(e.chart.update),e.setSize=function(t){return e.setMetricSize(e.metric,t)},e.setView=function(t){if(e.setMetricView(e.metric,t),"chart"===t)return e.showChart()},"chart"===e.metric.view&&e.showChart(),e.$on("metrics:data:update",function(t,r,n){return e.value=parseFloat(n[e.metric.id]),e.data[0].values.push({x:r,y:e.value}),e.data[0].values.length>e.window&&e.data[0].values.shift(),"chart"===e.metric.view&&e.showChart(),"chart"===e.metric.view&&e.chart.clearHighlights(),e.chart.tooltip.hidden(!0)}),t.find(".metric-title").qtip({content:{text:e.metric.id},position:{my:"bottom left",at:"top left"},style:{classes:"qtip-light qtip-timeline-bar"}})}}}),angular.module("flinkApp").service("MetricsService",["$http","$q","flinkConfig","$interval",funct
 ion(e,t,r,n){return this.metrics={},this.values={},this.watched={},this.observer={jobid:null,nodeid:null,
-callback:null},this.refresh=n(function(e){return function(){return angular.forEach(e.metrics,function(t,r){return angular.forEach(t,function(t,n){var i;if(i=[],angular.forEach(t,function(e,t){return i.push(e.id)}),i.length>0)return e.getMetrics(r,n,i).then(function(t){if(r===e.observer.jobid&&n===e.observer.nodeid&&e.observer.callback)return e.observer.callback(t)})})})}}(this),r["refresh-interval"]),this.registerObserver=function(e,t,r){return this.observer.jobid=e,this.observer.nodeid=t,this.observer.callback=r},this.unRegisterObserver=function(){return this.observer={jobid:null,nodeid:null,callback:null}},this.setupMetrics=function(e,t){return this.setupLS(),this.watched[e]=[],angular.forEach(t,function(t){return function(r,n){if(r.id)return t.watched[e].push(r.id)}}(this))},this.getWindow=function(){return 100},this.setupLS=function(){return null==sessionStorage.flinkMetrics&&this.saveSetup(),this.metrics=JSON.parse(sessionStorage.flinkMetrics)},this.saveSetup=function(){return 
 sessionStorage.flinkMetrics=JSON.stringify(this.metrics)},this.saveValue=function(e,t,r){if(null==this.values[e]&&(this.values[e]={}),null==this.values[e][t]&&(this.values[e][t]=[]),this.values[e][t].push(r),this.values[e][t].length>this.getWindow())return this.values[e][t].shift()},this.getValues=function(e,t,r){var n;return null==this.values[e]?[]:null==this.values[e][t]?[]:(n=[],angular.forEach(this.values[e][t],function(e){return function(e,t){if(null!=e.values[r])return n.push({x:e.timestamp,y:e.values[r]})}}(this)),n)},this.setupLSFor=function(e,t){if(null==this.metrics[e]&&(this.metrics[e]={}),null==this.metrics[e][t])return this.metrics[e][t]=[]},this.addMetric=function(e,t,r){return this.setupLSFor(e,t),this.metrics[e][t].push({id:r,size:"small",view:"chart"}),this.saveSetup()},this.removeMetric=function(e){return function(t,r,n){var i;if(null!=e.metrics[t][r])return i=e.metrics[t][r].indexOf(n),i===-1&&(i=_.findIndex(e.metrics[t][r],{id:n})),i!==-1&&e.metrics[t][r].splice(
 i,1),e.saveSetup()}}(this),this.setMetricSize=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:i,view:n.view}),e.saveSetup()}}(this),this.setMetricView=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:n.size,view:i}),e.saveSetup()}}(this),this.orderMetrics=function(e,t,r,n){return this.setupLSFor(e,t),angular.forEach(this.metrics[e][t],function(i){return function(o,s){if(o.id===r.id&&(i.metrics[e][t].splice(s,1),s<n))return n-=1}}(this)),this.metrics[e][t].splice(n,0,r),this.saveSetup()},this.getMetricsSetup=function(e){return function(t,r){return{names:_.map(e.metrics[t][r],function(e){return _.isString(e)?{id:e,size:"small",view:"chart"}:e})}}}(this),this.getAvailableMetrics=function(
 n){return function(i,o){var s;return n.setupLSFor(i,o),s=t.defer(),e.get(r.jobServer+"jobs/"+i+"/vertices/"+o+"/metrics").success(function(e){var t;return t=[],angular.forEach(e,function(e,r){var s;if(s=n.metrics[i][o].indexOf(e.id),s===-1&&(s=_.findIndex(n.metrics[i][o],{id:e.id})),s===-1)return t.push(e)}),s.resolve(t)}),s.promise}}(this),this.getAllAvailableMetrics=function(n){return function(n,i){var o;return o=t.defer(),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics").success(function(e){return o.resolve(e)}),o.promise}}(this),this.getMetrics=function(n,i,o){var s,a;return s=t.defer(),a=o.join(","),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics?get="+a).success(function(e){return function(t){var r,o;return o={},angular.forEach(t,function(e,t){return o[e.id]=parseInt(e.value)}),r={timestamp:Date.now(),values:o},e.saveValue(n,i,r),s.resolve(r)}}(this)),s.promise},this.setupLS(),this}]),angular.module("flinkApp").controller("JobManagerConfigController",["$scope","JobMa
 nagerConfigService",function(e,t){return t.loadConfig().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.config=t})}]).controller("JobManagerLogsController",["$scope","JobManagerLogsService",function(e,t){return t.loadLogs().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.log=t}),e.reloadData=function(){return t.loadLogs().then(function(t){return e.jobmanager.log=t})}}]).controller("JobManagerStdoutController",["$scope","JobManagerStdoutService",function(e,t){return t.loadStdout().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.stdout=t}),e.reloadData=function(){return t.loadStdout().then(function(t){return e.jobmanager.stdout=t})}}]),angular.module("flinkApp").service("JobManagerConfigService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"jobmanager/config").success(function(e,t,r,i){return i=e,n.resolve(e)}),n.promis
 e},this}]).service("JobManagerLogsService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadLogs=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/log").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]).service("JobManagerStdoutService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadStdout=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/stdout").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("OverviewController",["$scope","OverviewService","JobsService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.jobObserver=function(){return e.runningJobs=r.getJobs("running"),e.finishedJobs=r.getJobs("finished")},r.registerObserver(e.jobObserver),e.$on("$destroy",function(){return r.unRegisterObserver(e.jobObserver)}),e.jobObserver(),t.loadOverview().then(function(t){return e.overview=t}),o=n(function(){return t.loadOverv
 iew().then(function(t){return e.overview=t})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]),angular.module("flinkApp").service("OverviewService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadOverview=function(){var i;return i=r.defer(),e.get(t.jobServer+"overview").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("JobSubmitController",["$scope","JobSubmitService","$interval","flinkConfig","$state","$location",function(e,t,r,n,i,o){var s;return e.yarn=o.absUrl().indexOf("/proxy/application_")!==-1,e.loadList=function(){return t.loadJarList().then(function(t){return e.address=t.address,e.noaccess=t.error,e.jars=t.files})},e.defaultState=function(){return e.plan=null,e.error=null,e.state={selected:null,parallelism:"",savepointPath:"",allowNonRestoredState:!1,"entry-class":"","program-args":"","plan-button":"Show Plan","submit-button":"Submit","action-time":0}},e.defaultStat
 e(),e.uploader={},e.loadList(),s=r(function(){return e.loadList()},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(s)}),e.selectJar=function(t){return e.state.selected===t?e.defaultState():(e.defaultState(),e.state.selected=t)},e.deleteJar=function(r,n){return e.state.selected===n&&e.defaultState(),angular.element(r.currentTarget).removeClass("fa-remove").addClass("fa-spin fa-spinner"),t.deleteJar(n).then(function(e){if(angular.element(r.currentTarget).removeClass("fa-spin fa-spinner").addClass("fa-remove"),null!=e.error)return alert(e.error)})},e.loadEntryClass=function(t){return e.state["entry-class"]=t},e.getPlan=function(){var r;if("Show Plan"===e.state["plan-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submit",e.state["plan-button"]="Getting Plan",e.error=null,e.plan=null,t.getPlan(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"]}).then(fu
 nction(t){if(r===e.state["action-time"])return e.state["plan-button"]="Show Plan",e.error=t.error,e.plan=t.plan})},e.runJob=function(){var r;if("Submit"===e.state["submit-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submitting",e.state["plan-button"]="Show Plan",e.error=null,t.runJob(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"],savepointPath:e.state.savepointPath,allowNonRestoredState:e.state.allowNonRestoredState}).then(function(t){if(r===e.state["action-time"]&&(e.state["submit-button"]="Submit",e.error=t.error,null!=t.jobid))return i.go("single-job.plan.subtasks",{jobid:t.jobid})})["catch"](function(t){return e.state["submit-button"]="Submit",e.error=t})},e.nodeid=null,e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.$broadcast("reload")):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=n
 ull,e.accumulators=null)},e.clearFiles=function(){return e.uploader={}},e.uploadFiles=function(t){return e.uploader={},1===t.length?(e.uploader.file=t[0],e.uploader.upload=!0):e.uploader.error="Did ya forget to select a file?"},e.startUpload=function(){var t,r;return null!=e.uploader.file?(t=new FormData,t.append("jarfile",e.uploader.file),e.uploader.upload=!1,e.uploader.success="Initializing upload...",r=new XMLHttpRequest,r.upload.onprogress=function(t){return e.uploader.success=null,e.uploader.progress=parseInt(100*t.loaded/t.total)},r.upload.onerror=function(t){return e.uploader.progress=null,e.uploader.error="An error occurred while uploading your file"},r.upload.onload=function(t){return e.uploader.progress=null,e.uploader.success="Saving..."},r.onreadystatechange=function(){var t;if(4===r.readyState)return t=JSON.parse(r.responseText),null!=t.error?(e.uploader.error=t.error,e.uploader.success=null):e.uploader.success="Uploaded!"},r.open("POST",n.jobServer+"jars/upload"),r.sen
 d(t)):console.log("Unexpected Error. This should not happen")}}]).filter("getJarSelectClass",function(){return function(e,t){return e===t?"fa-check-square":"fa-square-o"}}),angular.module("flinkApp").service("JobSubmitService",["$http","flinkConfig","$q",function(e,t,r){return this.loadJarList=function(){var n;return n=r.defer(),e.get(t.jobServer+"jars/").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this.deleteJar=function(n){var i;return i=r.defer(),e["delete"](t.jobServer+"jars/"+encodeURIComponent(n)).success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.getPlan=function(n,i){var o;return o=r.defer(),e.get(t.jobServer+"jars/"+encodeURIComponent(n)+"/plan",{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this.runJob=function(n,i){var o;return o=r.defer(),e.post(t.jobServer+"jars/"+encodeURIComponent(n)+"/run",{},{params:i}).success(function(e,t,r,n){return o.resolve(e)}).error(function(e){return o.reject(e)}),o.promise},this}]),angular
 .module("flinkApp").controller("AllTaskManagersController",["$scope","TaskManagersService","$interval","flinkConfig",function(e,t,r,n){var i;return t.loadManagers().then(function(t){return e.managers=t}),i=r(function(){return t.loadManagers().then(function(t){return e.managers=t})},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(i)})}]).controller("SingleTaskManagerController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.metrics={},r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t}),o=n(function(){return r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]).controller("SingleTaskManagerLogsController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.log={},e.taskmanagerid=t.taskmanagerid,r.loadLogs(t.taskmanagerid).then(function(t){re
 turn e.log=t}),e.reloadData=function(){return r.loadLogs(t.taskmanagerid).then(function(t){return e.log=t})}}]).controller("SingleTaskManagerStdoutController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){return e.stdout={},e.taskmanagerid=t.taskmanagerid,r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t}),e.reloadData=function(){return r.loadStdout(t.taskmanagerid).then(function(t){return e.stdout=t})}}]),angular.module("flinkApp").service("TaskManagersService",["$http","flinkConfig","$q",function(e,t,r){return this.loadManagers=function(){var n;return n=r.defer(),e.get(t.jobServer+"taskmanagers").success(function(e,t,r,i){return n.resolve(e.taskmanagers)}),n.promise},this}]).service("SingleTaskManagerService",["$http","flinkConfig","$q",function(e,t,r){return this.loadMetrics=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n).success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.loa
 dLogs=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/log").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.loadStdout=function(n){var i;return i=r.defer(),e.get(t.jobServer+"taskmanagers/"+n+"/stdout").success(function(e,t,r,n){return i.resolve(e)}),i.promise},this}]);
\ No newline at end of file
+angular.module("flinkApp",["ui.router","angularMoment","dndLists"]).run(["$rootScope",function(e){return e.sidebarVisible=!1,e.showSidebar=function(){return e.sidebarVisible=!e.sidebarVisible,e.sidebarClass="force-show"}}]).value("flinkConfig",{jobServer:"","refresh-interval":1e4}).value("watermarksConfig",{noWatermark:-0x8000000000000000}).run(["JobsService","MainService","flinkConfig","$interval",function(e,t,r,n){return t.loadConfig().then(function(t){return angular.extend(r,t),e.listJobs(),n(function(){return e.listJobs()},r["refresh-interval"])})}]).config(["$uiViewScrollProvider",function(e){return e.useAnchorScroll()}]).run(["$rootScope","$state",function(e,t){return e.$on("$stateChangeStart",function(e,r,n,i){if(r.redirectTo)return e.preventDefault(),t.go(r.redirectTo,n)})}]).config(["$stateProvider","$urlRouterProvider",function(e,t){return e.state("completed-jobs",{url:"/completed-jobs",views:{main:{templateUrl:"partials/jobs/completed-jobs.html",controller:"CompletedJobsC
 ontroller"}}}).state("single-job",{url:"/jobs/{jobid}","abstract":!0,views:{main:{templateUrl:"partials/jobs/job.html",controller:"SingleJobController"}}}).state("single-job.plan",{url:"",redirectTo:"single-job.plan.subtasks",views:{details:{templateUrl:"partials/jobs/job.plan.html",controller:"JobPlanController"}}}).state("single-job.plan.subtasks",{url:"",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.subtasks.html",controller:"JobPlanSubtasksController"}}}).state("single-job.plan.metrics",{url:"/metrics",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.metrics.html",controller:"JobPlanMetricsController"}}}).state("single-job.plan.watermarks",{url:"/watermarks",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.watermarks.html"}}}).state("single-job.plan.taskmanagers",{url:"/taskmanagers",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.taskmanagers.html",controller:"JobPlanTaskManagersController"}}}).
 state("single-job.plan.accumulators",{url:"/accumulators",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.accumulators.html",controller:"JobPlanAccumulatorsController"}}}).state("single-job.plan.checkpoints",{url:"/checkpoints",redirectTo:"single-job.plan.checkpoints.overview",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.checkpoints.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.overview",{url:"/overview",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.overview.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.summary",{url:"/summary",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.summary.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.history",{url:"/history",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.history.html",controll
 er:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.config",{url:"/config",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.config.html",controller:"JobPlanCheckpointsController"}}}).state("single-job.plan.checkpoints.details",{url:"/details/{checkpointId}",views:{"checkpoints-view":{templateUrl:"partials/jobs/job.plan.node.checkpoints.details.html",controller:"JobPlanCheckpointDetailsController"}}}).state("single-job.plan.backpressure",{url:"/backpressure",views:{"node-details":{templateUrl:"partials/jobs/job.plan.node-list.backpressure.html",controller:"JobPlanBackPressureController"}}}).state("single-job.timeline",{url:"/timeline",views:{details:{templateUrl:"partials/jobs/job.timeline.html"}}}).state("single-job.timeline.vertex",{url:"/{vertexId}",views:{vertex:{templateUrl:"partials/jobs/job.timeline.vertex.html",controller:"JobTimelineVertexController"}}}).state("single-job.exceptions",{url:"/exceptions",views:{details:{templ
 ateUrl:"partials/jobs/job.exceptions.html",controller:"JobExceptionsController"}}}).state("single-job.config",{url:"/config",views:{details:{templateUrl:"partials/jobs/job.config.html"}}}),t.otherwise("/completed-jobs")}]),angular.module("flinkApp").directive("bsLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getLabelClass=function(){return"label label-"+e.translateLabelState(n.status)}}}}]).directive("bpLabel",["JobsService",function(e){return{transclude:!0,replace:!0,scope:{getBackPressureLabelClass:"&",status:"@"},template:"<span title='{{status}}' ng-class='getBackPressureLabelClass()'><ng-transclude></ng-transclude></span>",link:function(t,r,n){return t.getBackPressureLabelClass=function(){return"label label-"+e.translateBackPressureLabelState(n.status)}}}}]).directive("indicatorPrimary",["Jobs
 Service",function(e){return{replace:!0,scope:{getLabelClass:"&",status:"@"},template:"<i title='{{status}}' ng-class='getLabelClass()' />",link:function(t,r,n){return t.getLabelClass=function(){return"fa fa-circle indicator indicator-"+e.translateLabelState(n.status)}}}}]).directive("tableProperty",function(){return{replace:!0,scope:{value:"="},template:"<td title=\"{{value || 'None'}}\">{{value || 'None'}}</td>"}}),angular.module("flinkApp").filter("amDurationFormatExtended",["angularMomentConfig",function(e){var t;return t=function(e,t,r){return"undefined"==typeof e||null===e?"":moment.duration(e,t).format(r,{trim:!1})},t.$stateful=e.statefulFilters,t}]).filter("humanizeDuration",function(){return function(e,t){var r,n,i,o,s,a;return"undefined"==typeof e||null===e?"":(o=e%1e3,a=Math.floor(e/1e3),s=a%60,a=Math.floor(a/60),i=a%60,a=Math.floor(a/60),n=a%24,a=Math.floor(a/24),r=a,0===r?0===n?0===i?0===s?o+"ms":s+"s ":i+"m "+s+"s":t?n+"h "+i+"m":n+"h "+i+"m "+s+"s":t?r+"d "+n+"h":r+"d 
 "+n+"h "+i+"m "+s+"s")}}).filter("limit",function(){return function(e){return e.length>73&&(e=e.substring(0,35)+"..."+e.substring(e.length-35,e.length)),e}}).filter("humanizeText",function(){return function(e){return e?e.replace(/&gt;/g,">").replace(/<br\/>/g,""):""}}).filter("humanizeBytes",function(){return function(e){var t,r;return r=["B","KB","MB","GB","TB","PB","EB"],t=function(e,n){var i;return i=Math.pow(1024,n),e<i?(e/i).toFixed(2)+" "+r[n]:e<1e3*i?(e/i).toPrecision(3)+" "+r[n]:t(e,n+1)},"undefined"==typeof e||null===e?"":e<1e3?e+" B":t(e,1)}}).filter("toLocaleString",function(){return function(e){return e.toLocaleString()}}).filter("toUpperCase",function(){return function(e){return e.toUpperCase()}}).filter("percentage",function(){return function(e){return(100*e).toFixed(0)+"%"}}).filter("humanizeWatermark",["watermarksConfig",function(e){return function(t){return isNaN(t)||t<=e.noWatermark?"No Watermark":t}}]).filter("increment",function(){return function(e){return parseI
 nt(e)+1}}).filter("humanizeChartNumeric",["humanizeBytesFilter","humanizeDurationFilter",function(e,t){return function(r,n){var i;return i="",null!==r&&(i=/bytes/i.test(n.id)&&/persecond/i.test(n.id)?e(r)+" / s":/bytes/i.test(n.id)?e(r):/persecond/i.test(n.id)?r+" / s":/time/i.test(n.id)||/latency/i.test(n.id)?t(r,!0):r),i}}]).filter("humanizeChartNumericTitle",["humanizeDurationFilter",function(e){return function(t,r){var n;return n="",null!==t&&(n=/bytes/i.test(r.id)&&/persecond/i.test(r.id)?t+" Bytes / s":/bytes/i.test(r.id)?t+" Bytes":/persecond/i.test(r.id)?t+" / s":/time/i.test(r.id)||/latency/i.test(r.id)?e(t,!1):t),n}}]).filter("searchMetrics",function(){return function(e,t){var r,n;return n=new RegExp(t,"gi"),function(){var t,i,o;for(o=[],t=0,i=e.length;t<i;t++)r=e[t],r.id.match(n)&&o.push(r);return o}()}}),angular.module("flinkApp").service("MainService",["$http","flinkConfig","$q",function(e,t,r){return this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer
 +"config").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this}]),angular.module("flinkApp").controller("JobManagerConfigController",["$scope","JobManagerConfigService",function(e,t){return t.loadConfig().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.config=t})}]).controller("JobManagerLogsController",["$scope","JobManagerLogsService",function(e,t){return t.loadLogs().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.log=t}),e.reloadData=function(){return t.loadLogs().then(function(t){return e.jobmanager.log=t})}}]).controller("JobManagerStdoutController",["$scope","JobManagerStdoutService",function(e,t){return t.loadStdout().then(function(t){return null==e.jobmanager&&(e.jobmanager={}),e.jobmanager.stdout=t}),e.reloadData=function(){return t.loadStdout().then(function(t){return e.jobmanager.stdout=t})}}]),angular.module("flinkApp").service("JobManagerConfigService",["$http","flinkConfig","$q",function(e,t,r){var n;re
 turn n={},this.loadConfig=function(){var n;return n=r.defer(),e.get(t.jobServer+"jobmanager/config").success(function(e,t,r,i){return i=e,n.resolve(e)}),n.promise},this}]).service("JobManagerLogsService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadLogs=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/log").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]).service("JobManagerStdoutService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadStdout=function(){var i;return i=r.defer(),e.get(t.jobServer+"jobmanager/stdout").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("RunningJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("running")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver
 ()}]).controller("CompletedJobsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return e.jobObserver=function(){return e.jobs=n.getJobs("finished")},n.registerObserver(e.jobObserver),e.$on("$destroy",function(){return n.unRegisterObserver(e.jobObserver)}),e.jobObserver()}]).controller("SingleJobController",["$scope","$state","$stateParams","JobsService","MetricsService","$rootScope","flinkConfig","$interval","$q","watermarksConfig",function(e,t,r,n,i,o,s,a,l,u){var c,d;return e.jobid=r.jobid,e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats={},d=a(function(){return n.loadJob(r.jobid).then(function(t){return e.job=t,e.$broadcast("reload")})},s["refresh-interval"]),e.$on("$destroy",function(){return e.job=null,e.plan=null,e.watermarks={},e.vertices=null,e.backPressureOperatorStats=null,a.cancel(d)}),e.cancelJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Cancelling
 ..."),n.cancelJob(r.jobid).then(function(e){return{}})},e.stopJob=function(e){return angular.element(e.currentTarget).removeClass("btn").removeClass("btn-default").html("Stopping..."),n.stopJob(r.jobid).then(function(e){return{}})},n.loadJob(r.jobid).then(function(t){return e.job=t,e.vertices=t.vertices,e.plan=t.plan,i.setupMetrics(r.jobid,t.vertices)}),c=function(t){var r,n,o,s;return o=function(t){return function(t){var r,n,o,s;return r=l.defer(),o=e.job.jid,s=function(){var e,r,i;for(i=[],n=e=0,r=t.parallelism-1;0<=r?e<=r:e>=r;n=0<=r?++e:--e)i.push(n+".currentLowWatermark");return i}(),i.getMetrics(o,t.id,s).then(function(e){var t,n,i,o,s,a,l;i=NaN,l={},o=e.values;for(t in o)a=o[t],s=t.replace(".currentLowWatermark",""),l[s]=a,(isNaN(i)||a<i)&&(i=a);return n=!isNaN(i)&&i>u.noWatermark?i:NaN,r.resolve({lowWatermark:n,watermarks:l})}),r.promise}}(this),r=l.defer(),s={},n=t.length,angular.forEach(t,function(e){return function(e,t){var i;return i=e.id,o(e).then(function(e){if(s[i]=e,
 t>=n-1)return r.resolve(s)})}}(this)),r.promise},e.hasWatermark=function(t){return e.watermarks[t]&&!isNaN(e.watermarks[t].lowWatermark)},e.$watch("plan",function(t){if(t)return c(t.nodes).then(function(t){return e.watermarks=t})}),e.$on("reload",function(){if(e.plan)return c(e.plan.nodes).then(function(t){return e.watermarks=t})})}]).controller("JobPlanController",["$scope","$state","$stateParams","$window","JobsService",function(e,t,r,n,i){return e.nodeid=null,e.nodeUnfolded=!1,e.stateList=i.stateList(),e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null,e.$broadcast("reload"),e.$broadcast("node:change",e.nodeid)):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null)},e.deactivateNode=function(){return e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null,e.operatorCheckpointStats=null},e.toggleFold=function(
 ){return e.nodeUnfolded=!e.nodeUnfolded}}]).controller("JobPlanSubtasksController",["$scope","JobsService",function(e,t){var r;return e.aggregate=!1,r=function(){return e.aggregate?t.getTaskManagers(e.nodeid).then(function(t){return e.taskmanagers=t}):t.getSubtasks(e.nodeid).then(function(t){return e.subtasks=t})},!e.nodeid||e.vertex&&e.vertex.st||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanAccumulatorsController",["$scope","JobsService",function(e,t){var r;return r=function(){return t.getAccumulators(e.nodeid).then(function(t){return e.accumulators=t.main,e.subtaskAccumulators=t.subtasks})},!e.nodeid||e.vertex&&e.vertex.accumulators||r(),e.$on("reload",function(t){if(e.nodeid)return r()})}]).controller("JobPlanCheckpointsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return e.checkpointDetails={},e.checkpointDetails.id=-1,n.getCheckpointConfig().then(function(t){return e.checkpointConfig=t}),i=function(){return 
 n.getCheckpointStats().then(function(t){if(null!==t)return e.checkpointStats=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobPlanCheckpointDetailsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i,o;return e.subtaskDetails={},e.checkpointDetails.id=r.checkpointId,i=function(t){return n.getCheckpointDetails(t).then(function(t){return null!==t?e.checkpoint=t:e.unknown_checkpoint=!0})},o=function(t,r){return n.getCheckpointSubtaskDetails(t,r).then(function(t){if(null!==t)return e.subtaskDetails[r]=t})},i(r.checkpointId),e.nodeid&&o(r.checkpointId,e.nodeid),e.$on("reload",function(t){if(i(r.checkpointId),e.nodeid)return o(r.checkpointId,e.nodeid)}),e.$on("$destroy",function(){return e.checkpointDetails.id=-1})}]).controller("JobPlanBackPressureController",["$scope","JobsService",function(e,t){var r;return r=function(){if(e.now=Date.now(),e.nodeid)return t.getOperatorBackPressure(e.nodeid).then(function(t){return e.backPressureOperatorS
 tats[e.nodeid]=t})},r(),e.$on("reload",function(e){return r()})}]).controller("JobTimelineVertexController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){var i;return i=function(){return n.getVertex(r.vertexId).then(function(t){return e.vertex=t})},i(),e.$on("reload",function(e){return i()})}]).controller("JobExceptionsController",["$scope","$state","$stateParams","JobsService",function(e,t,r,n){return n.loadExceptions().then(function(t){return e.exceptions=t})}]).controller("JobPropertiesController",["$scope","JobsService",function(e,t){return e.changeNode=function(r){return r!==e.nodeid?(e.nodeid=r,t.getNode(r).then(function(t){return e.node=t})):(e.nodeid=null,e.node=null)}}]).controller("JobPlanMetricsController",["$scope","JobsService","MetricsService",function(e,t,r){var n,i;if(e.dragging=!1,e.window=r.getWindow(),e.availableMetrics=null,e.$on("$destroy",function(){return r.unRegisterObserver()}),i=function(){return t.getVertex(e.nodeid).then(function(t){re
 turn e.vertex=t}),r.getAvailableMetrics(e.jobid,e.nodeid).then(function(t){return e.availableMetrics=t.sort(n),e.metrics=r.getMetricsSetup(e.jobid,e.nodeid).names,r.registerObserver(e.jobid,e.nodeid,function(t){return e.$broadcast("metrics:data:update",t.timestamp,t.values)})})},n=function(e,t){var r,n;return r=e.id.toLowerCase(),n=t.id.toLowerCase(),r<n?-1:r>n?1:0},e.dropped=function(t,n,o,s,a){return r.orderMetrics(e.jobid,e.nodeid,o,n),e.$broadcast("metrics:refresh",o),i(),!1},e.dragStart=function(){return e.dragging=!0},e.dragEnd=function(){return e.dragging=!1},e.addMetric=function(t){return r.addMetric(e.jobid,e.nodeid,t.id),i()},e.removeMetric=function(t){return r.removeMetric(e.jobid,e.nodeid,t),i()},e.setMetricSize=function(t,n){return r.setMetricSize(e.jobid,e.nodeid,t,n),i()},e.setMetricView=function(t,n){return r.setMetricView(e.jobid,e.nodeid,t,n),i()},e.getValues=function(t){return r.getValues(e.jobid,e.nodeid,t)},e.$on("node:change",function(t,r){if(!e.dragging)return
  i()}),e.nodeid)return i()}]),angular.module("flinkApp").directive("vertex",["$state",function(e){return{template:"<svg class='timeline secondary' width='0' height='0'></svg>",scope:{data:"="},link:function(e,t,r){var n,i,o;o=t.children()[0],i=t.width(),angular.element(o).attr("width",i),(n=function(e){var t,r,n;return d3.select(o).selectAll("*").remove(),n=[],angular.forEach(e.subtasks,function(e,t){var r;return r=[{label:"Scheduled",color:"#666",borderColor:"#555",starting_time:e.timestamps.SCHEDULED,ending_time:e.timestamps.DEPLOYING,type:"regular"},{label:"Deploying",color:"#aaa",borderColor:"#555",starting_time:e.timestamps.DEPLOYING,ending_time:e.timestamps.RUNNING,type:"regular"}],e.timestamps.FINISHED>0&&r.push({label:"Running",color:"#ddd",borderColor:"#555",starting_time:e.timestamps.RUNNING,ending_time:e.timestamps.FINISHED,type:"regular"}),n.push({label:"("+e.subtask+") "+e.host,times:r})}),t=d3.timeline().stack().tickFormat({format:d3.time.format("%L"),tickSize:1}).pref
 ix("single").labelFormat(function(e){return e}).margin({left:100,right:0,top:0,bottom:0}).itemHeight(30).relativeTime(),r=d3.select(o).datum(n).call(t)})(e.data)}}}]).directive("timeline",["$state",function(e){return{template:"<svg class='timeline' width='0' height='0'></svg>",scope:{vertices:"=",jobid:"="},link:function(t,r,n){var i,o,s,a;s=r.children()[0],o=r.width(),angular.element(s).attr("width",o),a=function(e){return e.replace("&gt;",">")},i=function(r){var n,i,o;return d3.select(s).selectAll("*").remove(),o=[],angular.forEach(r,function(e){if(e["start-time"]>-1)return"scheduled"===e.type?o.push({times:[{label:a(e.name),color:"#cccccc",borderColor:"#555555",starting_time:e["start-time"],ending_time:e["end-time"],type:e.type}]}):o.push({times:[{label:a(e.name),color:"#d9f1f7",borderColor:"#62cdea",starting_time:e["start-time"],ending_time:e["end-time"],link:e.id,type:e.type}]})}),n=d3.timeline().stack().click(function(r,n,i){if(r.link)return e.go("single-job.timeline.vertex",{
 jobid:t.jobid,vertexId:r.link})}).tickFormat({format:d3.time.format("%L"),tickSize:1}).prefix("main").margin({left:0,right:0,top:0,bottom:0}).itemHeight(30).showBorderLine().showHourTimeline(),i=d3.select(s).datum(o).call(n)},t.$watch(n.vertices,function(e){if(e)return i(e)})}}}]).directive("split",function(){return{compile:function(e,t){return Split(e.children(),{sizes:[50,50],direction:"vertical"})}}}).directive("jobPlan",["$timeout",function(e){return{template:"<svg class='graph'><g /></svg> <svg class='tmp' width='1' height='1'><g /></svg> <div class='btn-group zoom-buttons'> <a class='btn btn-default zoom-in' ng-click='zoomIn()'><i class='fa fa-plus' /></a> <a class='btn btn-default zoom-out' ng-click='zoomOut()'><i class='fa fa-minus' /></a> </div>",scope:{plan:"=",watermarks:"=",setNode:"&"},link:function(e,t,r){var n,i,o,s,a,l,u,c,d,f,p,m,h,g,b,v,k,j,S,w,C,$,y,J,M;p=null,C=d3.behavior.zoom(),M=[],g=r.jobid,S=t.children()[0],j=t.children().children()[0],w=t.children()[1],l=d3
 .select(S),u=d3.select(j),c=d3.select(w),n=t.width(),angular.element(t.children()[0]).width(n),v=0,b=0,e.zoomIn=function(){var e,t,r;if(C.scale()<2.99)return e=C.translate(),t=e[0]*(C.scale()+.1/C.scale()),r=e[1]*(C.scale()+.1/C.scale()),C.scale(C.scale()+.1),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},e.zoomOut=function(){var e,t,r;if(C.scale()>.31)return C.scale(C.scale()-.1),e=C.translate(),t=e[0]*(C.scale()-.1/C.scale()),r=e[1]*(C.scale()-.1/C.scale()),C.translate([t,r]),u.attr("transform","translate("+t+","+r+") scale("+C.scale()+")"),v=C.scale(),b=C.translate()},o=function(e){var t;return t="",null==e.ship_strategy&&null==e.local_strategy||(t+="<div class='edge-label'>",null!=e.ship_strategy&&(t+=e.ship_strategy),void 0!==e.temp_mode&&(t+=" ("+e.temp_mode+")"),void 0!==e.local_strategy&&(t+=",<br>"+e.local_strategy),t+="</div>"),t},h=function(e){return"partialSolution"===e||"nextPartialSolution"===e||"workse
 t"===e||"nextWorkset"===e||"solutionSet"===e||"solutionDelta"===e},m=function(e,t){return"mirror"===t?"node-mirror":h(t)?"node-iteration":"node-normal"},s=function(e,t,r,n){var i,o;return i="<div href='#/jobs/"+g+"/vertex/"+e.id+"' class='node-label "+m(e,t)+"'>",i+="mirror"===t?"<h3 class='node-name'>Mirror of "+e.operator+"</h3>":"<h3 class='node-name'>"+e.operator+"</h3>",""===e.description?i+="":(o=e.description,o=J(o),i+="<h4 class='step-name'>"+o+"</h4>"),null!=e.step_function?i+=f(e.id,r,n):(h(t)&&(i+="<h5>"+t+" Node</h5>"),""!==e.parallelism&&(i+="<h5>Parallelism: "+e.parallelism+"</h5>"),void 0!==e.lowWatermark&&(i+="<h5>Low Watermark: "+e.lowWatermark+"</h5>"),void 0!==e.operator&&e.operator_strategy&&(i+="<h5>Operation: "+J(e.operator_strategy)+"</h5>")),i+="</div>"},f=function(e,t,r){var n,i;return i="svg-"+e,n="<svg class='"+i+"' width="+t+" height="+r+"><g /></svg>"},J=function(e){var t;for("<"===e.charAt(0)&&(e=e.replace("<","&lt;"),e=e.replace(">","&gt;")),t="";e.len
 gth>30;)t=t+e.substring(0,30)+"<br>",e=e.substring(30,e.length);return t+=e},a=function(e,t,r,n,i,o){return null==n&&(n=!1),r.id===t.partial_solution?e.setNode(r.id,{label:s(r,"partialSolution",i,o),labelType:"html","class":m(r,"partialSolution")}):r.id===t.next_partial_solution?e.setNode(r.id,{label:s(r,"nextPartialSolution",i,o),labelType:"html","class":m(r,"nextPartialSolution")}):r.id===t.workset?e.setNode(r.id,{label:s(r,"workset",i,o),labelType:"html","class":m(r,"workset")}):r.id===t.next_workset?e.setNode(r.id,{label:s(r,"nextWorkset",i,o),labelType:"html","class":m(r,"nextWorkset")}):r.id===t.solution_set?e.setNode(r.id,{label:s(r,"solutionSet",i,o),labelType:"html","class":m(r,"solutionSet")}):r.id===t.solution_delta?e.setNode(r.id,{label:s(r,"solutionDelta",i,o),labelType:"html","class":m(r,"solutionDelta")}):e.setNode(r.id,{label:s(r,"",i,o),labelType:"html","class":m(r,"")})},i=function(e,t,r,n,i){return e.setEdge(i.id,r.id,{label:o(i),labelType:"html",arrowhead:"normal
 "})},k=function(e,t){var r,n,o,s,l,u,d,f,p,m,h,g,b,v;for(n=[],null!=t.nodes?v=t.nodes:(v=t.step_function,o=!0),s=0,u=v.length;s<u;s++)if(r=v[s],p=0,f=0,r.step_function&&(b=new dagreD3.graphlib.Graph({multigraph:!0,compound:!0}).setGraph({nodesep:20,edgesep:0,ranksep:20,rankdir:"LR",marginx:10,marginy:10}),M[r.id]=b,k(b,r),h=new dagreD3.render,c.select("g").call(h,b),p=b.graph().width,f=b.graph().height,angular.element(w).empty()),a(e,t,r,o,p,f),n.push(r.id),null!=r.inputs)for(g=r.inputs,l=0,d=g.length;l<d;l++)m=g[l],i(e,t,r,n,m);return e},y=function(e,t){var r,n,i;for(n in e.nodes){if(r=e.nodes[n],r.id===t)return r;if(null!=r.step_function)for(i in r.step_function)if(r.step_function[i].id===t)return r.step_function[i]}},$=function(e,t){var r,n,i,o;if(!_.isEmpty(t))for(o=e.nodes,r=0,n=o.length;r<n;r++)i=o[r],t[i.id]&&!isNaN(t[i.id].lowWatermark)&&(i.lowWatermark=t[i.id].lowWatermark);return e},b=0,v=0,d=function(){var t,r,n,i,o,s;if(e.plan){p=new dagreD3.graphlib.Graph({multigraph:!0
 ,compound:!0}).setGraph({nodesep:70,edgesep:0,ranksep:50,rankdir:"LR",marginx:40,marginy:40}),k(p,$(e.plan,e.watermarks)),u.selectAll("*").remove(),u.attr("transform","scale(1)"),n=new dagreD3.render,u.call(n,p);for(t in M)i=M[t],l.select("svg.svg-"+t+" g").call(n,i);return r=.5,o=Math.floor((angular.element(S).width()-p.graph().width*r)/2),s=Math.floor((angular.element(S).height()-p.graph().height*r)/2),0!==v&&0!==b?(C.scale(v).translate(b),u.attr("transform","translate("+b+") scale("+v+")")):(C.scale(r).translate([o,s]),u.attr("transform","translate("+o+", "+s+") scale("+C.scale()+")")),C.on("zoom",function(){var e;return e=d3.event,v=e.scale,b=e.translate,u.attr("transform","translate("+b+") scale("+v+")")}),C(l),u.selectAll(".node").on("click",function(t){return e.setNode({nodeid:t})})}},e.$watch(r.plan,function(e){if(e)return d()}),e.$watch(r.watermarks,function(t){if(t&&e.plan)return d()})}}}]),angular.module("flinkApp").service("JobsService",["$http","flinkConfig","$log","amM
 oment","$q","$timeout",function(e,t,r,n,i,o){var s,a,l,u,c,d;return s=null,a=null,l={},c={running:[],finished:[],cancelled:[],failed:[]},u=[],d=function(){return angular.forEach(u,function(e){return e()})},this.registerObserver=function(e){return u.push(e)},this.unRegisterObserver=function(e){var t;return t=u.indexOf(e),u.splice(t,1)},this.stateList=function(){return["SCHEDULED","DEPLOYING","RUNNING","FINISHED","FAILED","CANCELING","CANCELED"]},this.translateLabelState=function(e){switch(e.toLowerCase()){case"finished":return"success";case"failed":return"danger";case"scheduled":return"default";case"deploying":return"info";case"running":return"primary";case"canceling":return"warning";case"pending":return"info";case"total":return"black";default:return"default"}},this.setEndTimes=function(e){return angular.forEach(e,function(e,t){if(!(e["end-time"]>-1))return e["end-time"]=e["start-time"]+e.duration})},this.processVertices=function(e){return angular.forEach(e.vertices,function(e,t){ret
 urn e.type="regular"}),e.vertices.unshift({name:"Scheduled","start-time":e.timestamps.CREATED,"end-time":e.timestamps.CREATED+1,type:"scheduled"})},this.listJobs=function(){var r;return r=i.defer(),e.get(t.jobServer+"jobs/overview").success(function(e){return function(t,n,i,o){return c.finished=[],c.running=[],_(t.jobs).groupBy(function(e){switch(e.state.toLowerCase()){case"finished":return"finished";case"failed":return"finished";case"canceled":return"finished";default:return"running"}}).forEach(function(t,r){switch(r){case"finished":return c.finished=e.setEndTimes(t);case"running":return c.running=e.setEndTimes(t)}}).value(),r.resolve(c),d()}}(this)),r.promise},this.getJobs=function(e){return c[e]},this.getAllJobs=function(){return c},this.loadJob=function(r){return s=null,l.job=i.defer(),e.get(t.jobServer+"jobs/"+r).success(function(n){return function(i,o,a,u){return n.setEndTimes(i.vertices),n.processVertices(i),e.get(t.jobServer+"jobs/"+r+"/config").success(function(e){return i=
 angular.extend(i,e),s=i,l.job.resolve(s)})}}(this)),l.job.promise},this.getNode=function(e){var t,r;return r=function(e,t){var n,i,o,s;for(n=0,i=t.length;n<i;n++){if(o=t[n],o.id===e)return o;if(o.step_function&&(s=r(e,o.step_function)),s)return s}return null},t=i.defer(),l.job.promise.then(function(n){return function(i){var o;return o=r(e,s.plan.nodes),o.vertex=n.seekVertex(e),t.resolve(o)}}(this)),t.promise},this.seekVertex=function(e){var t,r,n,i;for(n=s.vertices,t=0,r=n.length;t<r;t++)if(i=n[t],i.id===e)return i;return null},this.getVertex=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(o){var a;return a=i.seekVertex(r),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/subtasktimes").success(function(e){return a.subtasks=e.subtasks,n.resolve(a)})}}(this)),n.promise},this.getSubtasks=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r).success(function(e){var
  t;return t=e.subtasks,n.resolve(t)})}}(this)),n.promise},this.getTaskManagers=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/taskmanagers").success(function(e){var t;return t=e.taskmanagers,n.resolve(t)})}}(this)),n.promise},this.getAccumulators=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return console.log(s.jid),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/accumulators").success(function(i){var o;return o=i["user-accumulators"],e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/subtasks/accumulators").success(function(e){var t;return t=e.subtasks,n.resolve({main:o,subtasks:t})})})}}(this)),n.promise},this.getCheckpointConfig=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/config").success(function(e){return angular.equals({},e)?r.resolve(null):r.resolv
 e(e)})}}(this)),r.promise},this.getCheckpointStats=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints").success(function(e,t,n,i){return angular.equals({},e)?r.resolve(null):r.resolve(e)})}}(this)),r.promise},this.getCheckpointDetails=function(r){var n;return n=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/details/"+r).success(function(e){return angular.equals({},e)?n.resolve(null):n.resolve(e)})}}(this)),n.promise},this.getCheckpointSubtaskDetails=function(r,n){var o;return o=i.defer(),l.job.promise.then(function(i){return function(i){return e.get(t.jobServer+"jobs/"+s.jid+"/checkpoints/details/"+r+"/subtasks/"+n).success(function(e){return angular.equals({},e)?o.resolve(null):o.resolve(e)})}}(this)),o.promise},this.getOperatorBackPressure=function(r){var n;return n=i.defer(),e.get(t.jobServer+"jobs/"+s.jid+"/vertices/"+r+"/backp
 ressure").success(function(e){return function(e){return n.resolve(e)}}(this)),n.promise},this.translateBackPressureLabelState=function(e){switch(e.toLowerCase()){case"in-progress":return"danger";case"ok":return"success";case"low":return"warning";case"high":return"danger";default:return"default"}},this.loadExceptions=function(){var r;return r=i.defer(),l.job.promise.then(function(n){return function(n){return e.get(t.jobServer+"jobs/"+s.jid+"/exceptions").success(function(e){return s.exceptions=e,r.resolve(e)})}}(this)),r.promise},this.cancelJob=function(r){return e.get(t.jobServer+"jobs/"+r+"/yarn-cancel"),e.patch(t.jobServer+"jobs/"+r+"?mode=cancel")},this.stopJob=function(r){return e.get("jobs/"+r+"/yarn-stop"),e.patch(t.jobServer+"jobs/"+r+"?mode=stop")},this}]),angular.module("flinkApp").directive("metricsGraph",function(){return{template:'<div class="panel panel-default panel-metric"> <div class="panel-heading"> <span class="metric-title">{{metric.id}}</span> <div class="buttons
 "> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.size != \'big\'}]" ng-click="setSize(\'small\')">Small</button> <button type="button" ng-class="[btnClasses, {active: metric.size == \'big\'}]" ng-click="setSize(\'big\')">Big</button> </div> <a title="Remove" class="btn btn-default btn-xs remove" ng-click="removeMetric()"><i class="fa fa-close" /></a> </div> </div> <div class="panel-body"> <svg ng-if="metric.view == \'chart\'"/> <div ng-if="metric.view != \'chart\'"> <div class="metric-numeric" title="{{value | humanizeChartNumericTitle:metric}}">{{value | humanizeChartNumeric:metric}}</div> </div> </div> <div class="buttons"> <div class="btn-group"> <button type="button" ng-class="[btnClasses, {active: metric.view == \'chart\'}]" ng-click="setView(\'chart\')">Chart</button> <button type="button" ng-class="[btnClasses, {active: metric.view != \'chart\'}]" ng-click="setView(\'numeric\')">Numeric</button> </div> </div>',replace:!0,scope:{metric:"=
 ",window:"=",removeMetric:"&",setMetricSize:"=",setMetricView:"=",getValues:"&"},link:function(e,t,r){
+return e.btnClasses=["btn","btn-default","btn-xs"],e.value=null,e.data=[{values:e.getValues()}],e.options={x:function(e,t){return e.x},y:function(e,t){return e.y},xTickFormat:function(e){return d3.time.format("%H:%M:%S")(new Date(e))},yTickFormat:function(e){var t,r,n,i;for(r=!1,n=0,i=1,t=Math.abs(e);!r&&n<50;)Math.pow(10,n)<=t&&t<Math.pow(10,n+i)?r=!0:n+=i;return r&&n>6?e/Math.pow(10,n)+"E"+n:""+e}},e.showChart=function(){return d3.select(t.find("svg")[0]).datum(e.data).transition().duration(250).call(e.chart)},e.chart=nv.models.lineChart().options(e.options).showLegend(!1).margin({top:15,left:60,bottom:30,right:30}),e.chart.yAxis.showMaxMin(!1),e.chart.tooltip.hideDelay(0),e.chart.tooltip.contentGenerator(function(e){return"<p>"+d3.time.format("%H:%M:%S")(new Date(e.point.x))+" | "+e.point.y+"</p>"}),nv.utils.windowResize(e.chart.update),e.setSize=function(t){return e.setMetricSize(e.metric,t)},e.setView=function(t){if(e.setMetricView(e.metric,t),"chart"===t)return e.showChart()},
 "chart"===e.metric.view&&e.showChart(),e.$on("metrics:data:update",function(t,r,n){return e.value=parseFloat(n[e.metric.id]),e.data[0].values.push({x:r,y:e.value}),e.data[0].values.length>e.window&&e.data[0].values.shift(),"chart"===e.metric.view&&e.showChart(),"chart"===e.metric.view&&e.chart.clearHighlights(),e.chart.tooltip.hidden(!0)}),t.find(".metric-title").qtip({content:{text:e.metric.id},position:{my:"bottom left",at:"top left"},style:{classes:"qtip-light qtip-timeline-bar"}})}}}),angular.module("flinkApp").service("MetricsService",["$http","$q","flinkConfig","$interval",function(e,t,r,n){return this.metrics={},this.values={},this.watched={},this.observer={jobid:null,nodeid:null,callback:null},this.refresh=n(function(e){return function(){return angular.forEach(e.metrics,function(t,r){return angular.forEach(t,function(t,n){var i;if(i=[],angular.forEach(t,function(e,t){return i.push(e.id)}),i.length>0)return e.getMetrics(r,n,i).then(function(t){if(r===e.observer.jobid&&n===e.o
 bserver.nodeid&&e.observer.callback)return e.observer.callback(t)})})})}}(this),r["refresh-interval"]),this.registerObserver=function(e,t,r){return this.observer.jobid=e,this.observer.nodeid=t,this.observer.callback=r},this.unRegisterObserver=function(){return this.observer={jobid:null,nodeid:null,callback:null}},this.setupMetrics=function(e,t){return this.setupLS(),this.watched[e]=[],angular.forEach(t,function(t){return function(r,n){if(r.id)return t.watched[e].push(r.id)}}(this))},this.getWindow=function(){return 100},this.setupLS=function(){return null==sessionStorage.flinkMetrics&&this.saveSetup(),this.metrics=JSON.parse(sessionStorage.flinkMetrics)},this.saveSetup=function(){return sessionStorage.flinkMetrics=JSON.stringify(this.metrics)},this.saveValue=function(e,t,r){if(null==this.values[e]&&(this.values[e]={}),null==this.values[e][t]&&(this.values[e][t]=[]),this.values[e][t].push(r),this.values[e][t].length>this.getWindow())return this.values[e][t].shift()},this.getValues=fu
 nction(e,t,r){var n;return null==this.values[e]?[]:null==this.values[e][t]?[]:(n=[],angular.forEach(this.values[e][t],function(e){return function(e,t){if(null!=e.values[r])return n.push({x:e.timestamp,y:e.values[r]})}}(this)),n)},this.setupLSFor=function(e,t){if(null==this.metrics[e]&&(this.metrics[e]={}),null==this.metrics[e][t])return this.metrics[e][t]=[]},this.addMetric=function(e,t,r){return this.setupLSFor(e,t),this.metrics[e][t].push({id:r,size:"small",view:"chart"}),this.saveSetup()},this.removeMetric=function(e){return function(t,r,n){var i;if(null!=e.metrics[t][r])return i=e.metrics[t][r].indexOf(n),i===-1&&(i=_.findIndex(e.metrics[t][r],{id:n})),i!==-1&&e.metrics[t][r].splice(i,1),e.saveSetup()}}(this),this.setMetricSize=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:i,view:n.view}),e.saveSetup()}}(this),this.setMetricVie
 w=function(e){return function(t,r,n,i){var o;if(null!=e.metrics[t][r])return o=e.metrics[t][r].indexOf(n.id),o===-1&&(o=_.findIndex(e.metrics[t][r],{id:n.id})),o!==-1&&(e.metrics[t][r][o]={id:n.id,size:n.size,view:i}),e.saveSetup()}}(this),this.orderMetrics=function(e,t,r,n){return this.setupLSFor(e,t),angular.forEach(this.metrics[e][t],function(i){return function(o,s){if(o.id===r.id&&(i.metrics[e][t].splice(s,1),s<n))return n-=1}}(this)),this.metrics[e][t].splice(n,0,r),this.saveSetup()},this.getMetricsSetup=function(e){return function(t,r){return{names:_.map(e.metrics[t][r],function(e){return _.isString(e)?{id:e,size:"small",view:"chart"}:e})}}}(this),this.getAvailableMetrics=function(n){return function(i,o){var s;return n.setupLSFor(i,o),s=t.defer(),e.get(r.jobServer+"jobs/"+i+"/vertices/"+o+"/metrics").success(function(e){var t;return t=[],angular.forEach(e,function(e,r){var s;if(s=n.metrics[i][o].indexOf(e.id),s===-1&&(s=_.findIndex(n.metrics[i][o],{id:e.id})),s===-1)return t.p
 ush(e)}),s.resolve(t)}),s.promise}}(this),this.getAllAvailableMetrics=function(n){return function(n,i){var o;return o=t.defer(),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics").success(function(e){return o.resolve(e)}),o.promise}}(this),this.getMetrics=function(n,i,o){var s,a;return s=t.defer(),a=o.join(","),e.get(r.jobServer+"jobs/"+n+"/vertices/"+i+"/metrics?get="+a).success(function(e){return function(t){var r,o;return o={},angular.forEach(t,function(e,t){return o[e.id]=parseInt(e.value)}),r={timestamp:Date.now(),values:o},e.saveValue(n,i,r),s.resolve(r)}}(this)),s.promise},this.setupLS(),this}]),angular.module("flinkApp").controller("OverviewController",["$scope","OverviewService","JobsService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.jobObserver=function(){return e.runningJobs=r.getJobs("running"),e.finishedJobs=r.getJobs("finished")},r.registerObserver(e.jobObserver),e.$on("$destroy",function(){return r.unRegisterObserver(e.jobObserver)}),e.jobObserve
 r(),t.loadOverview().then(function(t){return e.overview=t}),o=n(function(){return t.loadOverview().then(function(t){return e.overview=t})},i["refresh-interval"]),e.$on("$destroy",function(){return n.cancel(o)})}]),angular.module("flinkApp").service("OverviewService",["$http","flinkConfig","$q",function(e,t,r){var n;return n={},this.loadOverview=function(){var i;return i=r.defer(),e.get(t.jobServer+"overview").success(function(e,t,r,o){return n=e,i.resolve(e)}),i.promise},this}]),angular.module("flinkApp").controller("JobSubmitController",["$scope","JobSubmitService","$interval","flinkConfig","$state","$location",function(e,t,r,n,i,o){var s;return e.yarn=o.absUrl().indexOf("/proxy/application_")!==-1,e.loadList=function(){return t.loadJarList().then(function(t){return e.address=t.address,e.noaccess=t.error,e.jars=t.files})},e.defaultState=function(){return e.plan=null,e.error=null,e.state={selected:null,parallelism:"",savepointPath:"",allowNonRestoredState:!1,"entry-class":"","progra
 m-args":"","plan-button":"Show Plan","submit-button":"Submit","action-time":0}},e.defaultState(),e.uploader={},e.loadList(),s=r(function(){return e.loadList()},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(s)}),e.selectJar=function(t){return e.state.selected===t?e.defaultState():(e.defaultState(),e.state.selected=t)},e.deleteJar=function(r,n){return e.state.selected===n&&e.defaultState(),angular.element(r.currentTarget).removeClass("fa-remove").addClass("fa-spin fa-spinner"),t.deleteJar(n).then(function(e){if(angular.element(r.currentTarget).removeClass("fa-spin fa-spinner").addClass("fa-remove"),null!=e.error)return alert(e.error)})},e.loadEntryClass=function(t){return e.state["entry-class"]=t},e.getPlan=function(){var r;if("Show Plan"===e.state["plan-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submit",e.state["plan-button"]="Getting Plan",e.error=null,e.plan=null,t.getPlan(e.state.selected,{"entry-class":e.state["e
 ntry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"]}).then(function(t){if(r===e.state["action-time"])return e.state["plan-button"]="Show Plan",e.error=t.error,e.plan=t.plan})},e.runJob=function(){var r;if("Submit"===e.state["submit-button"])return r=(new Date).getTime(),e.state["action-time"]=r,e.state["submit-button"]="Submitting",e.state["plan-button"]="Show Plan",e.error=null,t.runJob(e.state.selected,{"entry-class":e.state["entry-class"],parallelism:e.state.parallelism,"program-args":e.state["program-args"],savepointPath:e.state.savepointPath,allowNonRestoredState:e.state.allowNonRestoredState}).then(function(t){if(r===e.state["action-time"]&&(e.state["submit-button"]="Submit",e.error=t.error,null!=t.jobid))return i.go("single-job.plan.subtasks",{jobid:t.jobid})})["catch"](function(t){return e.state["submit-button"]="Submit",e.error=t})},e.nodeid=null,e.changeNode=function(t){return t!==e.nodeid?(e.nodeid=t,e.vertex=null,e.subtasks=null,e.accumula
 tors=null,e.$broadcast("reload")):(e.nodeid=null,e.nodeUnfolded=!1,e.vertex=null,e.subtasks=null,e.accumulators=null)},e.clearFiles=function(){return e.uploader={}},e.uploadFiles=function(t){return e.uploader={},1===t.length?(e.uploader.file=t[0],e.uploader.upload=!0):e.uploader.error="Did ya forget to select a file?"},e.startUpload=function(){var t,r;return null!=e.uploader.file?(t=new FormData,t.append("jarfile",e.uploader.file),e.uploader.upload=!1,e.uploader.success="Initializing upload...",r=new XMLHttpRequest,r.upload.onprogress=function(t){return e.uploader.success=null,e.uploader.progress=parseInt(100*t.loaded/t.total)},r.upload.onerror=function(t){return e.uploader.progress=null,e.uploader.error="An error occurred while uploading your file"},r.upload.onload=function(t){return e.uploader.progress=null,e.uploader.success="Saving..."},r.onreadystatechange=function(){var t;if(4===r.readyState)return t=JSON.parse(r.responseText),null!=t.error?(e.uploader.error=t.error,e.uploader
 .success=null):e.uploader.success="Uploaded!"},r.open("POST",n.jobServer+"jars/upload"),r.send(t)):console.log("Unexpected Error. This should not happen")}}]).filter("getJarSelectClass",function(){return function(e,t){return e===t?"fa-check-square":"fa-square-o"}}),angular.module("flinkApp").service("JobSubmitService",["$http","flinkConfig","$q",function(e,t,r){return this.loadJarList=function(){var n;return n=r.defer(),e.get(t.jobServer+"jars/").success(function(e,t,r,i){return n.resolve(e)}),n.promise},this.deleteJar=function(n){var i;return i=r.defer(),e["delete"](t.jobServer+"jars/"+encodeURIComponent(n)).success(function(e,t,r,n){return i.resolve(e)}),i.promise},this.getPlan=function(n,i){var o;return o=r.defer(),e.get(t.jobServer+"jars/"+encodeURIComponent(n)+"/plan",{params:i}).success(function(e,t,r,n){return o.resolve(e)}),o.promise},this.runJob=function(n,i){var o;return o=r.defer(),e.post(t.jobServer+"jars/"+encodeURIComponent(n)+"/run",{},{params:i}).success(function(e,t
 ,r,n){return o.resolve(e)}).error(function(e){return o.reject(e)}),o.promise},this}]),angular.module("flinkApp").controller("AllTaskManagersController",["$scope","TaskManagersService","$interval","flinkConfig",function(e,t,r,n){var i;return t.loadManagers().then(function(t){return e.managers=t}),i=r(function(){return t.loadManagers().then(function(t){return e.managers=t})},n["refresh-interval"]),e.$on("$destroy",function(){return r.cancel(i)})}]).controller("SingleTaskManagerController",["$scope","$stateParams","SingleTaskManagerService","$interval","flinkConfig",function(e,t,r,n,i){var o;return e.metrics={},r.loadMetrics(t.taskmanagerid).then(function(t){return e.metrics=t}),o=n(function(){return r.loadMetrics(t.taskmanagerid).t

<TRUNCATED>

[07/22] flink git commit: [hotfix] Change shutdown order in WebMonitorEndpoint to avoid illegal state

Posted by tr...@apache.org.
[hotfix] Change shutdown order in WebMonitorEndpoint to avoid illegal state


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/eeb453af
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/eeb453af
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/eeb453af

Branch: refs/heads/master
Commit: eeb453af12b5d49e8f49667f93ecc0e9e8093532
Parents: b3122f2
Author: Till Rohrmann <tr...@apache.org>
Authored: Tue Feb 6 19:11:53 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Thu Feb 15 08:44:18 2018 +0100

----------------------------------------------------------------------
 .../org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/eeb453af/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
index 021a009..dfcb134 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
@@ -575,8 +575,6 @@ public class WebMonitorEndpoint<T extends RestfulGateway> extends RestServerEndp
 
 	@Override
 	public void shutdown(Time timeout) {
-		super.shutdown(timeout);
-
 		executionGraphCache.close();
 
 		final File tmpDir = restConfiguration.getTmpDir();
@@ -593,6 +591,8 @@ public class WebMonitorEndpoint<T extends RestfulGateway> extends RestServerEndp
 		} catch (Exception e) {
 			log.warn("Error while stopping leaderElectionService", e);
 		}
+
+		super.shutdown(timeout);
 	}
 
 	//-------------------------------------------------------------------------


[20/22] flink git commit: [FLINK-8643] [flip6] Use JobManagerOptions#SLOT_REQUEST_TIMEOUT in ExecutionGraph

Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleWithCoLocationHintTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleWithCoLocationHintTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleWithCoLocationHintTest.java
index 19f4a2f..ecd88bc 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleWithCoLocationHintTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleWithCoLocationHintTest.java
@@ -22,6 +22,7 @@ import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.jobmaster.LogicalSlot;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
 
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -66,18 +67,18 @@ public class ScheduleWithCoLocationHintTest extends SchedulerTestBase {
 		CoLocationConstraint c6 = new CoLocationConstraint(ccg);
 
 		// schedule 4 tasks from the first vertex group
-		LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList()).get();
-		LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c2), false, Collections.emptyList()).get();
-		LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c3), false, Collections.emptyList()).get();
-		LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c4), false, Collections.emptyList()).get();
-		LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList()).get();
-		LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c2), false, Collections.emptyList()).get();
-		LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c3), false, Collections.emptyList()).get();
-		LogicalSlot s8 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c5), false, Collections.emptyList()).get();
-		LogicalSlot s9 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 5, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c6), false, Collections.emptyList()).get();
-		LogicalSlot s10 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c4), false, Collections.emptyList()).get();
-		LogicalSlot s11 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c5), false, Collections.emptyList()).get();
-		LogicalSlot s12 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 5, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c6), false, Collections.emptyList()).get();
+		LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c2), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c3), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c4), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c2), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c3), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s8 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c5), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s9 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 5, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c6), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s10 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c4), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s11 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c5), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s12 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 5, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c6), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 		assertNotNull(s1);
 		assertNotNull(s2);
@@ -127,7 +128,7 @@ public class ScheduleWithCoLocationHintTest extends SchedulerTestBase {
 		assertTrue(testingSlotProvider.getNumberOfAvailableSlots() >= 1);
 
 		LogicalSlot single = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertex(new JobVertexID(), 0, 1, null)), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertex(new JobVertexID(), 0, 1, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 		assertNotNull(single);
 
 		s1.releaseSlot();
@@ -164,11 +165,11 @@ public class ScheduleWithCoLocationHintTest extends SchedulerTestBase {
 		CoLocationConstraint c1 = new CoLocationConstraint(new CoLocationGroup());
 
 		LogicalSlot s1 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertex(jid1, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertex(jid1, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 		LogicalSlot s2 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertex(jid2, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertex(jid2, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
-		LogicalSlot sSolo = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 1, null)), false, Collections.emptyList()).get();
+		LogicalSlot sSolo = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 1, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 		ResourceID taskManager = s1.getTaskManagerLocation().getResourceID();
 
@@ -177,7 +178,7 @@ public class ScheduleWithCoLocationHintTest extends SchedulerTestBase {
 		sSolo.releaseSlot();
 
 		LogicalSlot sNew = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 		assertEquals(taskManager, sNew.getTaskManagerLocation().getResourceID());
 
 		assertEquals(2, testingSlotProvider.getNumberOfLocalizedAssignments());
@@ -200,14 +201,14 @@ public class ScheduleWithCoLocationHintTest extends SchedulerTestBase {
 		CoLocationConstraint c1 = new CoLocationConstraint(new CoLocationGroup());
 
 		LogicalSlot s1 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertex(jid1, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertex(jid1, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 		s1.releaseSlot();
 
-		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 1, null)), false, Collections.emptyList()).get();
-		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 2, null)), false, Collections.emptyList()).get();
+		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 1, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 2, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 		try {
-			testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList()).get();
+			testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			fail("Scheduled even though no resource was available.");
 		} catch (ExecutionException e) {
 			assertTrue(e.getCause() instanceof NoResourceAvailableException);
@@ -241,35 +242,35 @@ public class ScheduleWithCoLocationHintTest extends SchedulerTestBase {
 		SlotSharingGroup shareGroup = new SlotSharingGroup();
 
 		// first wave
-		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 		// second wave
 		LogicalSlot s21 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertex(jid2, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc1), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertex(jid2, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 		LogicalSlot s22 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertex(jid2, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc2), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertex(jid2, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc2), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 		LogicalSlot s23 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertex(jid2, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc3), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertex(jid2, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc3), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 		LogicalSlot s24 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertex(jid2, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc4), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertex(jid2, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc4), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 		// third wave
 		LogicalSlot s31 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertex(jid3, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc2), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertex(jid3, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc2), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 		LogicalSlot s32 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertex(jid3, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc3), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertex(jid3, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc3), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 		LogicalSlot s33 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertex(jid3, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc4), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertex(jid3, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc4), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 		LogicalSlot s34 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertex(jid3, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc1), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertex(jid3, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
-		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
-		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList()).get();
+		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 		assertEquals(s21.getTaskManagerLocation(), s34.getTaskManagerLocation());
 		assertEquals(s22.getTaskManagerLocation(), s31.getTaskManagerLocation());
@@ -301,25 +302,25 @@ public class ScheduleWithCoLocationHintTest extends SchedulerTestBase {
 
 		// schedule something into the shared group so that both instances are in the sharing group
 		LogicalSlot s1 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 		LogicalSlot s2 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get();
 
 		// schedule one locally to instance 1
 		LogicalSlot s3 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 
 		// schedule with co location constraint (yet unassigned) and a preference for
 		// instance 1, but it can only get instance 2
 		LogicalSlot s4 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 
 		// schedule something into the assigned co-location constraints and check that they override the
 		// other preferences
 		LogicalSlot s5 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid3, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc2)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid3, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get();
 		LogicalSlot s6 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid3, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid3, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 
 		// check that each slot got three
 		assertEquals(s1.getTaskManagerLocation(), s3.getTaskManagerLocation());
@@ -361,9 +362,9 @@ public class ScheduleWithCoLocationHintTest extends SchedulerTestBase {
 		CoLocationConstraint cc2 = new CoLocationConstraint(ccg);
 
 		LogicalSlot s1 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 		LogicalSlot s2 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc2)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get();
 
 		s1.releaseSlot();
 		s2.releaseSlot();
@@ -372,9 +373,9 @@ public class ScheduleWithCoLocationHintTest extends SchedulerTestBase {
 		assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots());
 
 		LogicalSlot s3 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc2)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get();
 		LogicalSlot s4 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 
 		// still preserves the previous instance mapping)
 		assertEquals(loc1, s3.getTaskManagerLocation());
@@ -408,9 +409,9 @@ public class ScheduleWithCoLocationHintTest extends SchedulerTestBase {
 		CoLocationConstraint cc2 = new CoLocationConstraint(ccg);
 
 		LogicalSlot s1 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 		LogicalSlot s2 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc2)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get();
 
 		s1.releaseSlot();
 		s2.releaseSlot();
@@ -419,13 +420,13 @@ public class ScheduleWithCoLocationHintTest extends SchedulerTestBase {
 		assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots());
 
 		LogicalSlot sa = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jidx, 0, 2, null)), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertexWithLocation(jidx, 0, 2, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 		LogicalSlot sb = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jidx, 1, 2, null)), false, Collections.emptyList()).get();
+				new ScheduledUnit(getTestVertexWithLocation(jidx, 1, 2, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 		try {
 			testingSlotProvider.allocateSlot(
-					new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc2)).get();
+					new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get();
 			fail("should not be able to find a resource");
 		}
 		catch (ExecutionException e) {
@@ -465,14 +466,14 @@ public class ScheduleWithCoLocationHintTest extends SchedulerTestBase {
 		// and give locality preferences that hint at using the same shared slot for both
 		// co location constraints (which we seek to prevent)
 		LogicalSlot s1 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 		LogicalSlot s2 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 
 		LogicalSlot s3 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 		LogicalSlot s4 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 
 		// check that each slot got three
 		assertEquals(s1.getTaskManagerLocation(), s3.getTaskManagerLocation());
@@ -514,14 +515,14 @@ public class ScheduleWithCoLocationHintTest extends SchedulerTestBase {
 		CoLocationConstraint cc2 = new CoLocationConstraint(ccg);
 
 		LogicalSlot s1 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 		LogicalSlot s2 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc2)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get();
 
 		LogicalSlot s3 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 		LogicalSlot s4 = testingSlotProvider.allocateSlot(
-				new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1)).get();
+				new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get();
 
 		// check that each slot got two
 		assertEquals(s1.getTaskManagerLocation(), s3.getTaskManagerLocation());

http://git-wip-us.apache.org/repos/asf/flink/blob/1321c170/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerIsolatedTasksTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerIsolatedTasksTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerIsolatedTasksTest.java
index ff166a8..86f5659 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerIsolatedTasksTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerIsolatedTasksTest.java
@@ -68,17 +68,17 @@ public class SchedulerIsolatedTasksTest extends SchedulerTestBase {
 		assertEquals(5, testingSlotProvider.getNumberOfAvailableSlots());
 
 		// schedule something into all slots
-		LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList()).get();
-		LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList()).get();
-		LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList()).get();
-		LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList()).get();
-		LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList()).get();
+		LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 		// the slots should all be different
 		assertTrue(areAllDistinct(s1, s2, s3, s4, s5));
 
 		try {
-			testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList()).get();
+			testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			fail("Scheduler accepted scheduling request without available resource.");
 		}
 		catch (ExecutionException e) {
@@ -91,8 +91,8 @@ public class SchedulerIsolatedTasksTest extends SchedulerTestBase {
 		assertEquals(2, testingSlotProvider.getNumberOfAvailableSlots());
 
 		// now we can schedule some more slots
-		LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList()).get();
-		LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList()).get();
+		LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
+		LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 		assertTrue(areAllDistinct(s1, s2, s3, s4, s5, s6, s7));
 
@@ -172,7 +172,7 @@ public class SchedulerIsolatedTasksTest extends SchedulerTestBase {
 		disposeThread.start();
 
 		for (int i = 0; i < NUM_TASKS_TO_SCHEDULE; i++) {
-			CompletableFuture<LogicalSlot> future = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), true, Collections.emptyList());
+			CompletableFuture<LogicalSlot> future = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), true, Collections.emptyList(), TestingUtils.infiniteTime());
 			future.thenAcceptAsync(
 				(LogicalSlot slot) -> {
 					synchronized (toRelease) {
@@ -207,11 +207,11 @@ public class SchedulerIsolatedTasksTest extends SchedulerTestBase {
 		final TaskManagerLocation taskManagerLocation3 = testingSlotProvider.addTaskManager(1);
 
 		List<LogicalSlot> slots = new ArrayList<>();
-		slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList()).get());
-		slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList()).get());
-		slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList()).get());
-		slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList()).get());
-		slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList()).get());
+		slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get());
+		slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get());
+		slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get());
+		slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get());
+		slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get());
 
 		testingSlotProvider.releaseTaskManager(taskManagerLocation2.getResourceID());
 
@@ -232,7 +232,7 @@ public class SchedulerIsolatedTasksTest extends SchedulerTestBase {
 
 		// cannot get another slot, since all instances are dead
 		try {
-			testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList()).get();
+			testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 			fail("Scheduler served a slot from a dead instance");
 		}
 		catch (ExecutionException e) {
@@ -254,7 +254,7 @@ public class SchedulerIsolatedTasksTest extends SchedulerTestBase {
 		final TaskManagerLocation taskManagerLocation3 = testingSlotProvider.addTaskManager(2);
 
 		// schedule something on an arbitrary instance
-		LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(new Instance[0])), false, Collections.emptyList()).get();
+		LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(new Instance[0])), false, Collections.emptyList(), TestingUtils.infiniteTime()).get();
 
 		// figure out how we use the location hints
 		ResourceID firstResourceId = s1.getTaskManagerLocation().getResourceID();
@@ -276,28 +276,28 @@ public class SchedulerIsolatedTasksTest extends SchedulerTestBase {
 		TaskManagerLocation third = taskManagerLocations.get((index + 2) % taskManagerLocations.size());
 
 		// something that needs to go to the first instance again
-		LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(s1.getTaskManagerLocation())), false, Collections.singleton(s1.getTaskManagerLocation())).get();
+		LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(s1.getTaskManagerLocation())), false, Collections.singleton(s1.getTaskManagerLocation()), TestingUtils.infiniteTime()).get();
 		assertEquals(first.getResourceID(), s2.getTaskManagerLocation().getResourceID());
 
 		// first or second --> second, because first is full
-		LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, second)), false, Arrays.asList(first, second)).get();
+		LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, second)), false, Arrays.asList(first, second), TestingUtils.infiniteTime()).get();
 		assertEquals(second.getResourceID(), s3.getTaskManagerLocation().getResourceID());
 
 		// first or third --> third (because first is full)
-		LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, Arrays.asList(first, third)).get();
-		LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, Arrays.asList(first, third)).get();
+		LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, Arrays.asList(first, third), TestingUtils.infiniteTime()).get();
+		LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, Arrays.asList(first, third), TestingUtils.infiniteTime()).get();
 		assertEquals(third.getResourceID(), s4.getTaskManagerLocation().getResourceID());
 		assertEquals(third.getResourceID(), s5.getTaskManagerLocation().getResourceID());
 
 		// first or third --> second, because all others are full
-		LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, Arrays.asList(first, third)).get();
+		LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, Arrays.asList(first, third), TestingUtils.infiniteTime()).get();
 		assertEquals(second.getResourceID(), s6.getTaskManagerLocation().getResourceID());
 
 		// release something on the first and second instance
 		s2.releaseSlot();
 		s6.releaseSlot();
 
-		LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, Arrays.asList(first, third)).get();
+		LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, Arrays.asList(first, third), TestingUtils.infiniteTime()).get();
 		assertEquals(first.getResourceID(), s7.getTaskManagerLocation().getResourceID());
 
 		assertEquals(1, testingSlotProvider.getNumberOfUnconstrainedAssignments());


[03/22] flink git commit: [FLINK-8605] [rest] Enable job cancellation from the web UI

Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java
index 96f57f8..39d9bea 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobTerminationHandler.java
@@ -35,6 +35,7 @@ import org.apache.flink.runtime.rest.messages.TerminationModeQueryParameter;
 import org.apache.flink.runtime.webmonitor.RestfulGateway;
 import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
 import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.Preconditions;
 
 import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus;
 
@@ -49,13 +50,18 @@ import java.util.concurrent.TimeoutException;
  */
 public class JobTerminationHandler extends AbstractRestHandler<RestfulGateway, EmptyRequestBody, EmptyResponseBody, JobTerminationMessageParameters> {
 
+	private final TerminationModeQueryParameter.TerminationMode defaultTerminationMode;
+
 	public JobTerminationHandler(
 			CompletableFuture<String> localRestAddress,
 			GatewayRetriever<? extends RestfulGateway> leaderRetriever,
 			Time timeout,
 			Map<String, String> headers,
-			MessageHeaders<EmptyRequestBody, EmptyResponseBody, JobTerminationMessageParameters> messageHeaders) {
+			MessageHeaders<EmptyRequestBody, EmptyResponseBody, JobTerminationMessageParameters> messageHeaders,
+			TerminationModeQueryParameter.TerminationMode defaultTerminationMode) {
 		super(localRestAddress, leaderRetriever, timeout, headers, messageHeaders);
+
+		this.defaultTerminationMode = Preconditions.checkNotNull(defaultTerminationMode);
 	}
 
 	@Override
@@ -65,7 +71,7 @@ public class JobTerminationHandler extends AbstractRestHandler<RestfulGateway, E
 		final TerminationModeQueryParameter.TerminationMode terminationMode;
 
 		if (terminationModes.isEmpty()) {
-			terminationMode = TerminationModeQueryParameter.TerminationMode.CANCEL;
+			terminationMode = defaultTerminationMode;
 		} else {
 			// picking the first termination mode value
 			terminationMode = terminationModes.get(0);

http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeaders.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeaders.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeaders.java
new file mode 100644
index 0000000..5333960
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeaders.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rest.messages;
+
+import org.apache.flink.runtime.rest.HttpMethodWrapper;
+import org.apache.flink.runtime.rest.handler.RestHandlerSpecification;
+import org.apache.flink.runtime.rest.handler.job.JobTerminationHandler;
+
+/**
+ * {@link RestHandlerSpecification} for the {@link JobTerminationHandler} which is registered for
+ * compatibility with the Yarn proxy as a GET call.
+ *
+ * <p>For more information @see <a href="https://issues.apache.org/jira/browse/YARN-2031">YARN-2031</a>.
+ *
+ * @deprecated This should be removed once we can send arbitrary REST calls via the Yarn proxy.
+ */
+@Deprecated
+public class YarnCancelJobTerminationHeaders implements RestHandlerSpecification {
+
+	private static final YarnCancelJobTerminationHeaders INSTANCE = new YarnCancelJobTerminationHeaders();
+
+	private static final String URL = String.format("/jobs/:%s/yarn-cancel", JobIDPathParameter.KEY);
+
+	private YarnCancelJobTerminationHeaders() {}
+
+	@Override
+	public HttpMethodWrapper getHttpMethod() {
+		return HttpMethodWrapper.GET;
+	}
+
+	@Override
+	public String getTargetRestEndpointURL() {
+		return URL;
+	}
+
+	public static YarnCancelJobTerminationHeaders getInstance() {
+		return INSTANCE;
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeaders.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeaders.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeaders.java
new file mode 100644
index 0000000..bbd36c0
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeaders.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rest.messages;
+
+import org.apache.flink.runtime.rest.HttpMethodWrapper;
+import org.apache.flink.runtime.rest.handler.RestHandlerSpecification;
+import org.apache.flink.runtime.rest.handler.job.JobTerminationHandler;
+
+/**
+ * {@link RestHandlerSpecification} for the {@link JobTerminationHandler} which is registered for
+ * compatibility with the Yarn proxy as a GET call.
+ *
+ * <p>For more information @see <a href="https://issues.apache.org/jira/browse/YARN-2031">YARN-2031</a>.
+ *
+ * @deprecated This should be removed once we can send arbitrary REST calls via the Yarn proxy.
+ */
+@Deprecated
+public class YarnStopJobTerminationHeaders implements RestHandlerSpecification {
+
+	private static final YarnStopJobTerminationHeaders INSTANCE = new YarnStopJobTerminationHeaders();
+
+	private static final String URL = String.format("/jobs/:%s/yarn-stop", JobIDPathParameter.KEY);
+
+	private YarnStopJobTerminationHeaders() {}
+
+	@Override
+	public HttpMethodWrapper getHttpMethod() {
+		return HttpMethodWrapper.GET;
+	}
+
+	@Override
+	public String getTargetRestEndpointURL() {
+		return URL;
+	}
+
+	public static YarnStopJobTerminationHeaders getInstance() {
+		return INSTANCE;
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
index 5aa3271..19efcbb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java
@@ -85,6 +85,9 @@ import org.apache.flink.runtime.rest.messages.JobVertexBackPressureHeaders;
 import org.apache.flink.runtime.rest.messages.JobVertexTaskManagersHeaders;
 import org.apache.flink.runtime.rest.messages.JobsOverviewHeaders;
 import org.apache.flink.runtime.rest.messages.SubtasksTimesHeaders;
+import org.apache.flink.runtime.rest.messages.TerminationModeQueryParameter;
+import org.apache.flink.runtime.rest.messages.YarnCancelJobTerminationHeaders;
+import org.apache.flink.runtime.rest.messages.YarnStopJobTerminationHeaders;
 import org.apache.flink.runtime.rest.messages.checkpoints.CheckpointConfigHeaders;
 import org.apache.flink.runtime.rest.messages.checkpoints.CheckpointStatisticDetailsHeaders;
 import org.apache.flink.runtime.rest.messages.checkpoints.CheckpointingStatisticsHeaders;
@@ -451,12 +454,21 @@ public class WebMonitorEndpoint<T extends RestfulGateway> extends RestServerEndp
 			responseHeaders,
 			JobVertexBackPressureHeaders.getInstance());
 
-		final JobTerminationHandler jobTerminationHandler = new JobTerminationHandler(
+		final JobTerminationHandler jobCancelTerminationHandler = new JobTerminationHandler(
 			restAddressFuture,
 			leaderRetriever,
 			timeout,
 			responseHeaders,
-			JobTerminationHeaders.getInstance());
+			JobTerminationHeaders.getInstance(),
+			TerminationModeQueryParameter.TerminationMode.CANCEL);
+
+		final JobTerminationHandler jobStopTerminationHandler = new JobTerminationHandler(
+			restAddressFuture,
+			leaderRetriever,
+			timeout,
+			responseHeaders,
+			JobTerminationHeaders.getInstance(),
+			TerminationModeQueryParameter.TerminationMode.STOP);
 
 		final File tmpDir = restConfiguration.getTmpDir();
 
@@ -504,7 +516,11 @@ public class WebMonitorEndpoint<T extends RestfulGateway> extends RestServerEndp
 		handlers.add(Tuple2.of(SubtaskCurrentAttemptDetailsHeaders.getInstance(), subtaskCurrentAttemptDetailsHandler));
 		handlers.add(Tuple2.of(JobVertexTaskManagersHeaders.getInstance(), jobVertexTaskManagersHandler));
 		handlers.add(Tuple2.of(JobVertexBackPressureHeaders.getInstance(), jobVertexBackPressureHandler));
-		handlers.add(Tuple2.of(JobTerminationHeaders.getInstance(), jobTerminationHandler));
+		handlers.add(Tuple2.of(JobTerminationHeaders.getInstance(), jobCancelTerminationHandler));
+
+		// TODO: Remove once the Yarn proxy can forward all REST verbs
+		handlers.add(Tuple2.of(YarnCancelJobTerminationHeaders.getInstance(), jobCancelTerminationHandler));
+		handlers.add(Tuple2.of(YarnStopJobTerminationHeaders.getInstance(), jobStopTerminationHandler));
 
 		optWebContent.ifPresent(
 			webContent -> handlers.add(Tuple2.of(WebContentHandlerSpecification.getInstance(), webContent)));

http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeadersTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeadersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeadersTest.java
new file mode 100644
index 0000000..f1a016b
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnCancelJobTerminationHeadersTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rest.messages;
+
+import org.apache.flink.runtime.rest.HttpMethodWrapper;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Tests for the {@link YarnCancelJobTerminationHeaders}.
+ */
+public class YarnCancelJobTerminationHeadersTest extends TestLogger {
+
+	// instance under test
+	private static final YarnCancelJobTerminationHeaders instance = YarnCancelJobTerminationHeaders.getInstance();
+
+	@Test
+	public void testMethod() {
+		assertThat(instance.getHttpMethod(), is(HttpMethodWrapper.GET));
+	}
+
+	@Test
+	public void testURL() {
+		assertThat(instance.getTargetRestEndpointURL(), endsWith("yarn-cancel"));
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/2a154173/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeadersTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeadersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeadersTest.java
new file mode 100644
index 0000000..a5d814f
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/YarnStopJobTerminationHeadersTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rest.messages;
+
+import org.apache.flink.runtime.rest.HttpMethodWrapper;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Tests for {@link YarnStopJobTerminationHeaders}.
+ */
+public class YarnStopJobTerminationHeadersTest extends TestLogger {
+
+	// instance under test
+	private static final YarnStopJobTerminationHeaders instance = YarnStopJobTerminationHeaders.getInstance();
+
+	@Test
+	public void testMethod() {
+		assertThat(instance.getHttpMethod(), is(HttpMethodWrapper.GET));
+	}
+
+	@Test
+	public void testURL() {
+		assertThat(instance.getTargetRestEndpointURL(), endsWith("yarn-stop"));
+	}
+
+}