You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2018/02/25 08:49:07 UTC

[1/3] flink git commit: [FLINK-8773] [flip6] Make JobManagerRunner shut down non blocking

Repository: flink
Updated Branches:
  refs/heads/master e461208a9 -> 4e7f03e41


[FLINK-8773] [flip6] Make JobManagerRunner shut down non blocking

The Dispatcher no longer shuts down the JobManagerRunner in a blocking fashion.
Instead it registers the termination futures and calls the shut down of the
JobManagerSharedServices once all JobManagerRunners have terminated.

This closes #5575.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2c40697e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2c40697e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2c40697e

Branch: refs/heads/master
Commit: 2c40697e46c0e4e9b7c5b86a3ef94b31c3330cb6
Parents: e461208
Author: Till Rohrmann <tr...@apache.org>
Authored: Sat Feb 24 15:39:31 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Sat Feb 24 23:19:09 2018 +0100

----------------------------------------------------------------------
 .../flink/runtime/dispatcher/Dispatcher.java    | 111 +++++++++----------
 .../runtime/jobmaster/JobManagerRunner.java     |  13 +--
 .../runtime/dispatcher/MiniDispatcherTest.java  |   1 +
 .../runtime/jobmaster/JobManagerRunnerTest.java |   8 +-
 4 files changed, 64 insertions(+), 69 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/2c40697e/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
index e212752..b31d04d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
@@ -65,13 +65,16 @@ import javax.annotation.Nullable;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
 
 /**
  * Base class for the Dispatcher component. The Dispatcher component is responsible
@@ -109,6 +112,8 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 	@Nullable
 	protected final String restAddress;
 
+	private CompletableFuture<Void> orphanedJobManagerRunnersTerminationFuture = CompletableFuture.completedFuture(null);
+
 	public Dispatcher(
 			RpcService rpcService,
 			String endpointId,
@@ -158,38 +163,39 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 	@Override
 	public CompletableFuture<Void> postStop() {
 		log.info("Stopping dispatcher {}.", getAddress());
-		Exception exception = null;
 
-		try {
-			clearState();
-		} catch (Exception e) {
-			exception = ExceptionUtils.firstOrSuppressed(e, exception);
-		}
+		final CompletableFuture<Void> jobManagerRunnersTerminationFuture = terminateJobManagerRunners();
 
-		try {
-			jobManagerSharedServices.shutdown();
-		} catch (Exception e) {
-			exception = ExceptionUtils.firstOrSuppressed(e, exception);
-		}
+		final CompletableFuture<Void> allJobManagerRunnersTerminationFuture = FutureUtils.completeAll(Arrays.asList(
+			jobManagerRunnersTerminationFuture,
+			orphanedJobManagerRunnersTerminationFuture));
 
-		try {
-			submittedJobGraphStore.stop();
-		} catch (Exception e) {
-			exception = ExceptionUtils.firstOrSuppressed(e, exception);
-		}
+		return FutureUtils.runAfterwards(
+			allJobManagerRunnersTerminationFuture,
+			() -> {
+				Exception exception = null;
+				try {
+					jobManagerSharedServices.shutdown();
+				} catch (Exception e) {
+					exception = ExceptionUtils.firstOrSuppressed(e, exception);
+				}
 
-		try {
-			leaderElectionService.stop();
-		} catch (Exception e) {
-			exception = ExceptionUtils.firstOrSuppressed(e, exception);
-		}
+				try {
+					submittedJobGraphStore.stop();
+				} catch (Exception e) {
+					exception = ExceptionUtils.firstOrSuppressed(e, exception);
+				}
 
-		if (exception != null) {
-			return FutureUtils.completedExceptionally(
-				new FlinkException("Could not properly terminate the Dispatcher.", exception));
-		} else {
-			return CompletableFuture.completedFuture(null);
-		}
+				try {
+					leaderElectionService.stop();
+				} catch (Exception e) {
+					exception = ExceptionUtils.firstOrSuppressed(e, exception);
+				}
+
+				if (exception != null) {
+					throw exception;
+				}
+			});
 	}
 
 	@Override
@@ -491,7 +497,8 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 		JobManagerRunner jobManagerRunner = jobManagerRunners.remove(jobId);
 
 		if (jobManagerRunner != null) {
-			jobManagerRunner.shutdown();
+			final CompletableFuture<Void> jobManagerRunnerTerminationFuture = jobManagerRunner.closeAsync();
+			registerOrphanedJobManagerTerminationFuture(jobManagerRunnerTerminationFuture);
 		}
 
 		if (cleanupHA) {
@@ -502,28 +509,17 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 	}
 
 	/**
-	 * Clears the state of the dispatcher.
+	 * Terminate all currently running {@link JobManagerRunner}.
 	 *
-	 * <p>The state are all currently running jobs.
+	 * @return Future which is completed once all {@link JobManagerRunner} have terminated
 	 */
-	private void clearState() throws Exception {
-		Exception exception = null;
-
+	private CompletableFuture<Void> terminateJobManagerRunners() {
 		log.info("Stopping all currently running jobs of dispatcher {}.", getAddress());
-		// stop all currently running JobManager since they run in the same process
-		for (JobManagerRunner jobManagerRunner : jobManagerRunners.values()) {
-			try {
-				jobManagerRunner.shutdown();
-			} catch (Exception e) {
-				exception = ExceptionUtils.firstOrSuppressed(e, exception);
-			}
-		}
+		final List<CompletableFuture<Void>> terminationFutures = jobManagerRunners.values().stream()
+			.map(JobManagerRunner::closeAsync)
+			.collect(Collectors.toList());
 
-		jobManagerRunners.clear();
-
-		if (exception != null) {
-			throw exception;
-		}
+		return FutureUtils.completeAll(terminationFutures);
 	}
 
 	/**
@@ -600,6 +596,12 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 		}
 	}
 
+	private void registerOrphanedJobManagerTerminationFuture(CompletableFuture<Void> jobManagerRunnerTerminationFuture) {
+		orphanedJobManagerRunnersTerminationFuture = FutureUtils.completeAll(Arrays.asList(
+			orphanedJobManagerRunnersTerminationFuture,
+			jobManagerRunnerTerminationFuture));
+	}
+
 	//------------------------------------------------------
 	// Leader contender
 	//------------------------------------------------------
@@ -619,11 +621,9 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 
 				// clear the state if we've been the leader before
 				if (getFencingToken() != null) {
-					try {
-						clearState();
-					} catch (Exception e) {
-						log.warn("Could not properly clear the Dispatcher state while granting leadership.", e);
-					}
+					final CompletableFuture<Void> jobManagerRunnersTerminationFuture = terminateJobManagerRunners();
+					registerOrphanedJobManagerTerminationFuture(jobManagerRunnersTerminationFuture);
+					jobManagerRunners.clear();
 				}
 
 				setFencingToken(dispatcherId);
@@ -644,11 +644,10 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 		runAsyncWithoutFencing(
 			() -> {
 				log.info("Dispatcher {} was revoked leadership.", getAddress());
-				try {
-					clearState();
-				} catch (Exception e) {
-					log.warn("Could not properly clear the Dispatcher state while revoking leadership.", e);
-				}
+
+				final CompletableFuture<Void> jobManagerRunnersTerminationFuture = terminateJobManagerRunners();
+				registerOrphanedJobManagerTerminationFuture(jobManagerRunnersTerminationFuture);
+				jobManagerRunners.clear();
 
 				// clear the fencing token indicating that we don't have the leadership right now
 				setFencingToken(null);

http://git-wip-us.apache.org/repos/asf/flink/blob/2c40697e/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
index 4269243..80aa673 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
@@ -40,6 +40,7 @@ import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup;
 import org.apache.flink.runtime.metrics.util.MetricUtils;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.util.AutoCloseableAsync;
 import org.apache.flink.util.ExceptionUtils;
 import org.apache.flink.util.FlinkException;
 
@@ -59,7 +60,7 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
  * The runner for the job manager. It deals with job level leader election and make underlying job manager
  * properly reacted.
  */
-public class JobManagerRunner implements LeaderContender, OnCompletionActions, FatalErrorHandler {
+public class JobManagerRunner implements LeaderContender, OnCompletionActions, FatalErrorHandler, AutoCloseableAsync {
 
 	private static final Logger log = LoggerFactory.getLogger(JobManagerRunner.class);
 
@@ -98,9 +99,6 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions, F
 	 * Exceptions that occur while creating the JobManager or JobManagerRunner are directly
 	 * thrown and not reported to the given {@code FatalErrorHandler}.
 	 *
-	 * <p>This JobManagerRunner assumes that it owns the given {@code JobManagerSharedServices}.
-	 * It will shut them down on error and on calls to {@link #shutdown()}.
-	 *
 	 * @throws Exception Thrown if the runner cannot be set up, because either one of the
 	 *                   required services could not be started, ot the Job could not be initialized.
 	 */
@@ -214,11 +212,8 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions, F
 		}
 	}
 
-	public void shutdown() throws Exception {
-		shutdownInternally().get();
-	}
-
-	private CompletableFuture<Void> shutdownInternally() {
+	@Override
+	public CompletableFuture<Void> closeAsync() {
 		synchronized (lock) {
 			if (!shutdown) {
 				shutdown = true;

http://git-wip-us.apache.org/repos/asf/flink/blob/2c40697e/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java
index 2b98939..c6eda2e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java
@@ -289,6 +289,7 @@ public class MiniDispatcherTest extends TestLogger {
 
 			final JobManagerRunner mock = mock(JobManagerRunner.class);
 			when(mock.getResultFuture()).thenReturn(resultFuture);
+			when(mock.closeAsync()).thenReturn(CompletableFuture.completedFuture(null));
 
 			return mock;
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/2c40697e/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java
index ba72293..0c238ce 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java
@@ -152,7 +152,7 @@ public class JobManagerRunnerTest extends TestLogger {
 
 			assertThat(resultFuture.get(), is(archivedExecutionGraph));
 		} finally {
-			jobManagerRunner.shutdown();
+			jobManagerRunner.close();
 		}
 	}
 
@@ -176,7 +176,7 @@ public class JobManagerRunnerTest extends TestLogger {
 				assertThat(ExceptionUtils.stripExecutionException(ee), instanceOf(JobNotFinishedException.class));
 			}
 		} finally {
-			jobManagerRunner.shutdown();
+			jobManagerRunner.close();
 		}
 	}
 
@@ -191,7 +191,7 @@ public class JobManagerRunnerTest extends TestLogger {
 
 			assertThat(resultFuture.isDone(), is(false));
 
-			jobManagerRunner.shutdown();
+			jobManagerRunner.closeAsync();
 
 			try {
 				resultFuture.get();
@@ -200,7 +200,7 @@ public class JobManagerRunnerTest extends TestLogger {
 				assertThat(ExceptionUtils.stripExecutionException(ee), instanceOf(JobNotFinishedException.class));
 			}
 		} finally {
-			jobManagerRunner.shutdown();
+			jobManagerRunner.close();
 		}
 	}
 


[2/3] flink git commit: [FLINK-8774] [flip6] Make shut down of ResourceManagerRunner non blocking

Posted by tr...@apache.org.
[FLINK-8774] [flip6] Make shut down of ResourceManagerRunner non blocking


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c9787b69
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c9787b69
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c9787b69

Branch: refs/heads/master
Commit: c9787b6992f05b429edb0678050a1273900d1a63
Parents: 2c40697
Author: Till Rohrmann <tr...@apache.org>
Authored: Sat Feb 24 16:24:25 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Sat Feb 24 23:19:33 2018 +0100

----------------------------------------------------------------------
 .../flink/runtime/minicluster/MiniCluster.java  |  2 +-
 .../resourcemanager/ResourceManagerRunner.java  | 32 ++++++++------------
 2 files changed, 14 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/c9787b69/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
index 01be01d..5b1fddb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
@@ -437,7 +437,7 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
 
 		if (resourceManagerRunner != null) {
 			try {
-				resourceManagerRunner.shutDown();
+				resourceManagerRunner.close();
 			} catch (Throwable t) {
 				exception = ExceptionUtils.firstOrSuppressed(t, exception);
 			}

http://git-wip-us.apache.org/repos/asf/flink/blob/c9787b69/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
index 26316ec..9daf96e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
@@ -20,26 +20,26 @@ package org.apache.flink.runtime.resourcemanager;
 
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.FutureUtils;
 import org.apache.flink.runtime.entrypoint.ClusterInformation;
 import org.apache.flink.runtime.heartbeat.HeartbeatServices;
 import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
 import org.apache.flink.runtime.metrics.MetricRegistry;
 import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.AutoCloseableAsync;
 import org.apache.flink.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
 
 /**
  * Simple {@link StandaloneResourceManager} runner. It instantiates the resource manager's services
  * and handles fatal errors by shutting the resource manager down.
  */
-public class ResourceManagerRunner implements FatalErrorHandler {
+public class ResourceManagerRunner implements FatalErrorHandler, AutoCloseableAsync {
 
 	private static final Logger LOG = LoggerFactory.getLogger(ResourceManagerRunner.class);
 
@@ -101,24 +101,18 @@ public class ResourceManagerRunner implements FatalErrorHandler {
 		resourceManager.start();
 	}
 
-	public void shutDown() throws Exception {
-		// wait for the completion
-		shutDownInternally().get();
-	}
-
-	private CompletableFuture<Void> shutDownInternally() {
+	@Override
+	public CompletableFuture<Void> closeAsync() {
 		synchronized (lock) {
 			resourceManager.shutDown();
 
-			return resourceManager.getTerminationFuture()
-				.thenAccept(
-					ignored -> {
-						try {
-							resourceManagerRuntimeServices.shutDown();
-						} catch (Exception e) {
-							throw new CompletionException(new FlinkException("Could not properly shut down the resource manager runtime services.", e));
-						}
-					});
+			return FutureUtils.runAfterwards(
+				resourceManager.getTerminationFuture(),
+				() -> {
+					synchronized (lock) {
+						resourceManagerRuntimeServices.shutDown();
+					}
+				});
 		}
 	}
 
@@ -130,7 +124,7 @@ public class ResourceManagerRunner implements FatalErrorHandler {
 	public void onFatalError(Throwable exception) {
 		LOG.error("Encountered fatal error.", exception);
 
-		CompletableFuture<Void> shutdownFuture = shutDownInternally();
+		CompletableFuture<Void> shutdownFuture = closeAsync();
 
 		shutdownFuture.whenComplete(
 			(Void ignored, Throwable throwable) -> {


[3/3] flink git commit: [FLINK-8775] [flip6] Non blocking MiniCluster shut down

Posted by tr...@apache.org.
[FLINK-8775] [flip6] Non blocking MiniCluster shut down

This closes #5576.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4e7f03e4
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4e7f03e4
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4e7f03e4

Branch: refs/heads/master
Commit: 4e7f03e418ced740a731c9323dabef7a594db43c
Parents: c9787b6
Author: Till Rohrmann <tr...@apache.org>
Authored: Sat Feb 24 16:20:58 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Sat Feb 24 23:44:53 2018 +0100

----------------------------------------------------------------------
 .../flink/runtime/dispatcher/Dispatcher.java    |  25 +-
 .../flink/runtime/minicluster/MiniCluster.java  | 334 +++++++++++--------
 .../runtime/minicluster/MiniClusterITCase.java  |   4 +-
 .../Flip6LocalStreamEnvironment.java            |   2 +-
 4 files changed, 215 insertions(+), 150 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/4e7f03e4/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
index b31d04d..bb60277 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
@@ -254,16 +254,22 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 
 				jobManagerRunner.getResultFuture().whenCompleteAsync(
 					(ArchivedExecutionGraph archivedExecutionGraph, Throwable throwable) -> {
-						if (archivedExecutionGraph != null) {
-							jobReachedGloballyTerminalState(archivedExecutionGraph);
-						} else {
-							final Throwable strippedThrowable = ExceptionUtils.stripCompletionException(throwable);
-
-							if (strippedThrowable instanceof JobNotFinishedException) {
-								jobNotFinished(jobId);
+						// check if we are still the active JobManagerRunner by checking the identity
+						//noinspection ObjectEquality
+						if (jobManagerRunner == jobManagerRunners.get(jobId)) {
+							if (archivedExecutionGraph != null) {
+								jobReachedGloballyTerminalState(archivedExecutionGraph);
 							} else {
-								onFatalError(new FlinkException("JobManagerRunner for job " + jobId + " failed.", strippedThrowable));
+								final Throwable strippedThrowable = ExceptionUtils.stripCompletionException(throwable);
+
+								if (strippedThrowable instanceof JobNotFinishedException) {
+									jobNotFinished(jobId);
+								} else {
+									onFatalError(new FlinkException("JobManagerRunner for job " + jobId + " failed.", strippedThrowable));
+								}
 							}
+						} else {
+							log.debug("There is a newer JobManagerRunner for the job {}.", jobId);
 						}
 					}, getMainThreadExecutor());
 
@@ -294,6 +300,9 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
 
 	@Override
 	public CompletableFuture<Collection<JobID>> listJobs(Time timeout) {
+		if (jobManagerRunners.isEmpty()) {
+			System.out.println("empty");
+		}
 		return CompletableFuture.completedFuture(
 			Collections.unmodifiableSet(new HashSet<>(jobManagerRunners.keySet())));
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/4e7f03e4/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
index 5b1fddb..3bd17d1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
@@ -69,14 +69,18 @@ import com.typesafe.config.Config;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nonnull;
 import javax.annotation.concurrent.GuardedBy;
 
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
 import java.util.concurrent.ExecutionException;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -97,6 +101,8 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
 
 	private final Time rpcTimeout;
 
+	private CompletableFuture<Void> terminationFuture;
+
 	@GuardedBy("lock")
 	private MetricRegistryImpl metricRegistry;
 
@@ -161,6 +167,7 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
 		this.miniClusterConfiguration = checkNotNull(miniClusterConfiguration, "config may not be null");
 
 		this.rpcTimeout = Time.seconds(10L);
+		this.terminationFuture = CompletableFuture.completedFuture(null);
 		running = false;
 	}
 
@@ -195,6 +202,9 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
 			LOG.info("Starting Flink Mini Cluster");
 			LOG.debug("Using configuration {}", miniClusterConfiguration);
 
+			// create a new termination future
+			terminationFuture = new CompletableFuture<>();
+
 			final Configuration configuration = miniClusterConfiguration.getConfiguration();
 			final Time rpcTimeout = miniClusterConfiguration.getRpcTimeout();
 			final int numTaskManagers = miniClusterConfiguration.getNumTaskManagers();
@@ -349,7 +359,7 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
 			catch (Exception e) {
 				// cleanup everything
 				try {
-					shutdownInternally();
+					close();
 				} catch (Exception ee) {
 					e.addSuppressed(ee);
 				}
@@ -370,141 +380,70 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
 	 * <p>This method shuts down all started services and components,
 	 * even if an exception occurs in the process of shutting down some component.
 	 *
-	 * @throws Exception Thrown, if the shutdown did not complete cleanly.
+	 * @return Future which is completed once the MiniCluster has been completely shut down
 	 */
-	public void shutdown() throws Exception {
+	@Override
+	public CompletableFuture<Void> closeAsync() {
 		synchronized (lock) {
 			if (running) {
 				LOG.info("Shutting down Flink Mini Cluster");
 				try {
-					shutdownInternally();
-				} finally {
-					running = false;
-				}
-				LOG.info("Flink Mini Cluster is shut down");
-			}
-		}
-	}
-
-	@GuardedBy("lock")
-	private void shutdownInternally() throws Exception {
-		// this should always be called under the lock
-		assert Thread.holdsLock(lock);
-
-		// collect the first exception, but continue and add all successive
-		// exceptions as suppressed
-		Throwable exception = null;
+					final int numComponents = 2 + miniClusterConfiguration.getNumTaskManagers();
+					final Collection<CompletableFuture<Void>> componentTerminationFutures = new ArrayList<>(numComponents);
 
-		// cancel all jobs and shut down the job dispatcher
-		if (dispatcher != null) {
-			try {
-				RpcUtils.terminateRpcEndpoint(dispatcher, rpcTimeout);
-			} catch (Exception e) {
-				exception = e;
-			}
-			dispatcher = null;
-		}
-
-		if (dispatcherRestEndpoint != null) {
-			try {
-				dispatcherRestEndpoint.shutDownAsync().get();
-			} catch (Exception e) {
-				exception = ExceptionUtils.firstOrSuppressed(e, exception);
-			}
-
-			dispatcherRestEndpoint = null;
-		}
-
-		if (resourceManagerLeaderRetriever != null) {
-			try {
-				resourceManagerLeaderRetriever.stop();
-			} catch (Exception e) {
-				exception = ExceptionUtils.firstOrSuppressed(e, exception);
-			}
-
-			resourceManagerLeaderRetriever = null;
-		}
+					componentTerminationFutures.add(shutDownDispatcher());
 
-		if (dispatcherLeaderRetriever != null) {
-			try {
-				dispatcherLeaderRetriever.stop();
-			} catch (Exception e) {
-				exception = ExceptionUtils.firstOrSuppressed(e, exception);
-			}
-
-			dispatcherLeaderRetriever = null;
-		}
-
-		if (resourceManagerRunner != null) {
-			try {
-				resourceManagerRunner.close();
-			} catch (Throwable t) {
-				exception = ExceptionUtils.firstOrSuppressed(t, exception);
-			}
-		}
-
-		if (taskManagers != null) {
-			for (TaskExecutor tm : taskManagers) {
-				if (tm != null) {
-					try {
-						tm.shutDown();
-						// wait for the TaskManager to properly terminate
-						tm.getTerminationFuture().get();
-					} catch (Throwable t) {
-						exception = ExceptionUtils.firstOrSuppressed(t, exception);
+					if (resourceManagerRunner != null) {
+						componentTerminationFutures.add(resourceManagerRunner.closeAsync());
+						resourceManagerRunner = null;
 					}
-				}
-			}
-			taskManagers = null;
-		}
-		// metrics shutdown
-		if (metricRegistry != null) {
-			metricRegistry.shutdown();
-			metricRegistry = null;
-		}
-
-		// shut down the RpcServices
-		exception = shutDownRpc(commonRpcService, exception);
-		exception = shutDownRpc(jobManagerRpcService, exception);
-		exception = shutDownRpcs(taskManagerRpcServices, exception);
-		exception = shutDownRpc(resourceManagerRpcService, exception);
-		commonRpcService = null;
-		jobManagerRpcService = null;
-		taskManagerRpcServices = null;
-		resourceManagerRpcService = null;
-
-		if (blobCacheService != null) {
-			try {
-				blobCacheService.close();
-			} catch (Exception e) {
-				exception = ExceptionUtils.firstOrSuppressed(e, exception);
-			}
-			blobCacheService = null;
-		}
 
-		// shut down the blob server
-		if (blobServer != null) {
-			try {
-				blobServer.close();
-			} catch (Exception e) {
-				exception = ExceptionUtils.firstOrSuppressed(e, exception);
-			}
-			blobServer = null;
-		}
+					if (taskManagers != null) {
+						for (TaskExecutor tm : taskManagers) {
+							if (tm != null) {
+								tm.shutDown();
+								componentTerminationFutures.add(tm.getTerminationFuture());
+							}
+						}
+						taskManagers = null;
+					}
 
-		// shut down high-availability services
-		if (haServices != null) {
-			try {
-				haServices.closeAndCleanupAllData();
-			} catch (Exception e) {
-				exception = ExceptionUtils.firstOrSuppressed(e, exception);
+					final FutureUtils.ConjunctFuture<Void> componentsTerminationFuture = FutureUtils.completeAll(componentTerminationFutures);
+
+					final CompletableFuture<Void> metricRegistryTerminationFuture = FutureUtils.runAfterwards(
+						componentsTerminationFuture,
+						() -> {
+							synchronized (lock) {
+								// metrics shutdown
+								if (metricRegistry != null) {
+									metricRegistry.shutdown();
+									metricRegistry = null;
+								}
+							}
+						});
+
+					// shut down the RpcServices
+					final CompletableFuture<Void> rpcServicesTerminationFuture = metricRegistryTerminationFuture
+						.thenCompose((Void ignored) -> terminateRpcServices());
+
+					final CompletableFuture<Void> remainingServicesTerminationFuture = FutureUtils.runAfterwards(
+						rpcServicesTerminationFuture,
+						this::terminateMiniClusterServices);
+
+						remainingServicesTerminationFuture.whenComplete(
+							(Void ignored, Throwable throwable) -> {
+								if (throwable != null) {
+									terminationFuture.completeExceptionally(ExceptionUtils.stripCompletionException(throwable));
+								} else {
+									terminationFuture.complete(null);
+								}
+							});
+				} finally {
+					running = false;
+				}
 			}
-			haServices = null;
-		}
 
-		// if anything went wrong, throw the first error with all the additional suppressed exceptions
-		if (exception != null) {
-			ExceptionUtils.rethrowException(exception, "Error while shutting down mini cluster");
+			return terminationFuture;
 		}
 	}
 
@@ -704,6 +643,137 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
 	}
 
 	// ------------------------------------------------------------------------
+	//  Internal methods
+	// ------------------------------------------------------------------------
+
+	@GuardedBy("lock")
+	private CompletableFuture<Void> shutDownDispatcher() {
+
+		final Collection<CompletableFuture<Void>> terminationFutures = new ArrayList<>(2);
+
+		// cancel all jobs and shut down the job dispatcher
+		if (dispatcher != null) {
+			dispatcher.shutDown();
+			terminationFutures.add(dispatcher.getTerminationFuture());
+
+			dispatcher = null;
+		}
+
+		if (dispatcherRestEndpoint != null) {
+			terminationFutures.add(dispatcherRestEndpoint.shutDownAsync());
+
+			dispatcherRestEndpoint = null;
+		}
+
+		final FutureUtils.ConjunctFuture<Void> dispatcherTerminationFuture = FutureUtils.completeAll(terminationFutures);
+
+		return FutureUtils.runAfterwards(
+			dispatcherTerminationFuture,
+			() -> {
+				Exception exception = null;
+
+				synchronized (lock) {
+					if (resourceManagerLeaderRetriever != null) {
+						try {
+							resourceManagerLeaderRetriever.stop();
+						} catch (Exception e) {
+							exception = ExceptionUtils.firstOrSuppressed(e, exception);
+						}
+
+						resourceManagerLeaderRetriever = null;
+					}
+
+					if (dispatcherLeaderRetriever != null) {
+						try {
+							dispatcherLeaderRetriever.stop();
+						} catch (Exception e) {
+							exception = ExceptionUtils.firstOrSuppressed(e, exception);
+						}
+
+						dispatcherLeaderRetriever = null;
+					}
+				}
+
+				if (exception != null) {
+					throw exception;
+				}
+			});
+	}
+
+	private void terminateMiniClusterServices() throws Exception {
+		// collect the first exception, but continue and add all successive
+		// exceptions as suppressed
+		Exception exception = null;
+
+		synchronized (lock) {
+			if (blobCacheService != null) {
+				try {
+					blobCacheService.close();
+				} catch (Exception e) {
+					exception = ExceptionUtils.firstOrSuppressed(e, exception);
+				}
+				blobCacheService = null;
+			}
+
+			// shut down the blob server
+			if (blobServer != null) {
+				try {
+					blobServer.close();
+				} catch (Exception e) {
+					exception = ExceptionUtils.firstOrSuppressed(e, exception);
+				}
+				blobServer = null;
+			}
+
+			// shut down high-availability services
+			if (haServices != null) {
+				try {
+					haServices.closeAndCleanupAllData();
+				} catch (Exception e) {
+					exception = ExceptionUtils.firstOrSuppressed(e, exception);
+				}
+				haServices = null;
+			}
+
+			if (exception != null) {
+				throw exception;
+			}
+		}
+	}
+
+	@Nonnull
+	private CompletionStage<Void> terminateRpcServices() {
+		final int numRpcServices;
+		if (miniClusterConfiguration.getRpcServiceSharing() == MiniClusterConfiguration.RpcServiceSharing.SHARED) {
+			numRpcServices = 1;
+		} else {
+			numRpcServices = 1 + 2 + miniClusterConfiguration.getNumTaskManagers(); // common, JM, RM, TMs
+		}
+
+		final Collection<CompletableFuture<?>> rpcTerminationFutures = new ArrayList<>(numRpcServices);
+
+		synchronized (lock) {
+			rpcTerminationFutures.add(commonRpcService.stopService());
+
+			if (miniClusterConfiguration.getRpcServiceSharing() != MiniClusterConfiguration.RpcServiceSharing.SHARED) {
+				rpcTerminationFutures.add(jobManagerRpcService.stopService());
+				rpcTerminationFutures.add(resourceManagerRpcService.stopService());
+
+				for (RpcService taskManagerRpcService : taskManagerRpcServices) {
+					rpcTerminationFutures.add(taskManagerRpcService.stopService());
+				}
+			}
+
+			commonRpcService = null;
+			jobManagerRpcService = null;
+			taskManagerRpcServices = null;
+			resourceManagerRpcService = null;
+		}
+
+		return FutureUtils.completeAll(rpcTerminationFutures);
+	}
+
+	// ------------------------------------------------------------------------
 	//  miscellaneous utilities
 	// ------------------------------------------------------------------------
 
@@ -743,16 +813,6 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
 		return priorException;
 	}
 
-	@Override
-	public CompletableFuture<Void> closeAsync() {
-		try {
-			shutdown();
-			return CompletableFuture.completedFuture(null);
-		} catch (Exception e) {
-			return FutureUtils.completedExceptionally(e);
-		}
-	}
-
 	private class TerminatingFatalErrorHandler implements FatalErrorHandler {
 
 		private final int index;
@@ -784,11 +844,7 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
 		@Override
 		public void onFatalError(Throwable exception) {
 			LOG.warn("Error in MiniCluster. Shutting the MiniCluster down.", exception);
-			try {
-				shutdown();
-			} catch (Exception e) {
-				LOG.warn("Could not shut down the MiniCluster.", e);
-			}
+			closeAsync();
 		}
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/4e7f03e4/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
index d974998..646827a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
@@ -67,7 +67,7 @@ public class MiniClusterITCase extends TestLogger {
 			executeJob(miniCluster);
 		}
 		finally {
-			miniCluster.shutdown();
+			miniCluster.close();
 		}
 	}
 
@@ -84,7 +84,7 @@ public class MiniClusterITCase extends TestLogger {
 			executeJob(miniCluster);
 		}
 		finally {
-			miniCluster.shutdown();
+			miniCluster.close();
 		}
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/4e7f03e4/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java
index 5941ee5..4cc23fc 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java
@@ -105,7 +105,7 @@ public class Flip6LocalStreamEnvironment extends LocalStreamEnvironment {
 		}
 		finally {
 			transformations.clear();
-			miniCluster.shutdown();
+			miniCluster.close();
 		}
 	}
 }