You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2018/02/25 08:49:07 UTC
[1/3] flink git commit: [FLINK-8773] [flip6] Make JobManagerRunner
shut down non blocking
Repository: flink
Updated Branches:
refs/heads/master e461208a9 -> 4e7f03e41
[FLINK-8773] [flip6] Make JobManagerRunner shut down non blocking
The Dispatcher no longer shuts down the JobManagerRunner in a blocking fashion.
Instead it registers the termination futures and calls the shut down of the
JobManagerSharedServices once all JobManagerRunners have terminated.
This closes #5575.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2c40697e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2c40697e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2c40697e
Branch: refs/heads/master
Commit: 2c40697e46c0e4e9b7c5b86a3ef94b31c3330cb6
Parents: e461208
Author: Till Rohrmann <tr...@apache.org>
Authored: Sat Feb 24 15:39:31 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Sat Feb 24 23:19:09 2018 +0100
----------------------------------------------------------------------
.../flink/runtime/dispatcher/Dispatcher.java | 111 +++++++++----------
.../runtime/jobmaster/JobManagerRunner.java | 13 +--
.../runtime/dispatcher/MiniDispatcherTest.java | 1 +
.../runtime/jobmaster/JobManagerRunnerTest.java | 8 +-
4 files changed, 64 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2c40697e/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
index e212752..b31d04d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
@@ -65,13 +65,16 @@ import javax.annotation.Nullable;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
/**
* Base class for the Dispatcher component. The Dispatcher component is responsible
@@ -109,6 +112,8 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
@Nullable
protected final String restAddress;
+ private CompletableFuture<Void> orphanedJobManagerRunnersTerminationFuture = CompletableFuture.completedFuture(null);
+
public Dispatcher(
RpcService rpcService,
String endpointId,
@@ -158,38 +163,39 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
@Override
public CompletableFuture<Void> postStop() {
log.info("Stopping dispatcher {}.", getAddress());
- Exception exception = null;
- try {
- clearState();
- } catch (Exception e) {
- exception = ExceptionUtils.firstOrSuppressed(e, exception);
- }
+ final CompletableFuture<Void> jobManagerRunnersTerminationFuture = terminateJobManagerRunners();
- try {
- jobManagerSharedServices.shutdown();
- } catch (Exception e) {
- exception = ExceptionUtils.firstOrSuppressed(e, exception);
- }
+ final CompletableFuture<Void> allJobManagerRunnersTerminationFuture = FutureUtils.completeAll(Arrays.asList(
+ jobManagerRunnersTerminationFuture,
+ orphanedJobManagerRunnersTerminationFuture));
- try {
- submittedJobGraphStore.stop();
- } catch (Exception e) {
- exception = ExceptionUtils.firstOrSuppressed(e, exception);
- }
+ return FutureUtils.runAfterwards(
+ allJobManagerRunnersTerminationFuture,
+ () -> {
+ Exception exception = null;
+ try {
+ jobManagerSharedServices.shutdown();
+ } catch (Exception e) {
+ exception = ExceptionUtils.firstOrSuppressed(e, exception);
+ }
- try {
- leaderElectionService.stop();
- } catch (Exception e) {
- exception = ExceptionUtils.firstOrSuppressed(e, exception);
- }
+ try {
+ submittedJobGraphStore.stop();
+ } catch (Exception e) {
+ exception = ExceptionUtils.firstOrSuppressed(e, exception);
+ }
- if (exception != null) {
- return FutureUtils.completedExceptionally(
- new FlinkException("Could not properly terminate the Dispatcher.", exception));
- } else {
- return CompletableFuture.completedFuture(null);
- }
+ try {
+ leaderElectionService.stop();
+ } catch (Exception e) {
+ exception = ExceptionUtils.firstOrSuppressed(e, exception);
+ }
+
+ if (exception != null) {
+ throw exception;
+ }
+ });
}
@Override
@@ -491,7 +497,8 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
JobManagerRunner jobManagerRunner = jobManagerRunners.remove(jobId);
if (jobManagerRunner != null) {
- jobManagerRunner.shutdown();
+ final CompletableFuture<Void> jobManagerRunnerTerminationFuture = jobManagerRunner.closeAsync();
+ registerOrphanedJobManagerTerminationFuture(jobManagerRunnerTerminationFuture);
}
if (cleanupHA) {
@@ -502,28 +509,17 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
}
/**
- * Clears the state of the dispatcher.
+ * Terminate all currently running {@link JobManagerRunner}.
*
- * <p>The state are all currently running jobs.
+ * @return Future which is completed once all {@link JobManagerRunner} have terminated
*/
- private void clearState() throws Exception {
- Exception exception = null;
-
+ private CompletableFuture<Void> terminateJobManagerRunners() {
log.info("Stopping all currently running jobs of dispatcher {}.", getAddress());
- // stop all currently running JobManager since they run in the same process
- for (JobManagerRunner jobManagerRunner : jobManagerRunners.values()) {
- try {
- jobManagerRunner.shutdown();
- } catch (Exception e) {
- exception = ExceptionUtils.firstOrSuppressed(e, exception);
- }
- }
+ final List<CompletableFuture<Void>> terminationFutures = jobManagerRunners.values().stream()
+ .map(JobManagerRunner::closeAsync)
+ .collect(Collectors.toList());
- jobManagerRunners.clear();
-
- if (exception != null) {
- throw exception;
- }
+ return FutureUtils.completeAll(terminationFutures);
}
/**
@@ -600,6 +596,12 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
}
}
+ private void registerOrphanedJobManagerTerminationFuture(CompletableFuture<Void> jobManagerRunnerTerminationFuture) {
+ orphanedJobManagerRunnersTerminationFuture = FutureUtils.completeAll(Arrays.asList(
+ orphanedJobManagerRunnersTerminationFuture,
+ jobManagerRunnerTerminationFuture));
+ }
+
//------------------------------------------------------
// Leader contender
//------------------------------------------------------
@@ -619,11 +621,9 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
// clear the state if we've been the leader before
if (getFencingToken() != null) {
- try {
- clearState();
- } catch (Exception e) {
- log.warn("Could not properly clear the Dispatcher state while granting leadership.", e);
- }
+ final CompletableFuture<Void> jobManagerRunnersTerminationFuture = terminateJobManagerRunners();
+ registerOrphanedJobManagerTerminationFuture(jobManagerRunnersTerminationFuture);
+ jobManagerRunners.clear();
}
setFencingToken(dispatcherId);
@@ -644,11 +644,10 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
runAsyncWithoutFencing(
() -> {
log.info("Dispatcher {} was revoked leadership.", getAddress());
- try {
- clearState();
- } catch (Exception e) {
- log.warn("Could not properly clear the Dispatcher state while revoking leadership.", e);
- }
+
+ final CompletableFuture<Void> jobManagerRunnersTerminationFuture = terminateJobManagerRunners();
+ registerOrphanedJobManagerTerminationFuture(jobManagerRunnersTerminationFuture);
+ jobManagerRunners.clear();
// clear the fencing token indicating that we don't have the leadership right now
setFencingToken(null);
http://git-wip-us.apache.org/repos/asf/flink/blob/2c40697e/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
index 4269243..80aa673 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
@@ -40,6 +40,7 @@ import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup;
import org.apache.flink.runtime.metrics.util.MetricUtils;
import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.util.AutoCloseableAsync;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FlinkException;
@@ -59,7 +60,7 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
* The runner for the job manager. It deals with job level leader election and make underlying job manager
* properly reacted.
*/
-public class JobManagerRunner implements LeaderContender, OnCompletionActions, FatalErrorHandler {
+public class JobManagerRunner implements LeaderContender, OnCompletionActions, FatalErrorHandler, AutoCloseableAsync {
private static final Logger log = LoggerFactory.getLogger(JobManagerRunner.class);
@@ -98,9 +99,6 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions, F
* Exceptions that occur while creating the JobManager or JobManagerRunner are directly
* thrown and not reported to the given {@code FatalErrorHandler}.
*
- * <p>This JobManagerRunner assumes that it owns the given {@code JobManagerSharedServices}.
- * It will shut them down on error and on calls to {@link #shutdown()}.
- *
* @throws Exception Thrown if the runner cannot be set up, because either one of the
* required services could not be started, ot the Job could not be initialized.
*/
@@ -214,11 +212,8 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions, F
}
}
- public void shutdown() throws Exception {
- shutdownInternally().get();
- }
-
- private CompletableFuture<Void> shutdownInternally() {
+ @Override
+ public CompletableFuture<Void> closeAsync() {
synchronized (lock) {
if (!shutdown) {
shutdown = true;
http://git-wip-us.apache.org/repos/asf/flink/blob/2c40697e/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java
index 2b98939..c6eda2e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java
@@ -289,6 +289,7 @@ public class MiniDispatcherTest extends TestLogger {
final JobManagerRunner mock = mock(JobManagerRunner.class);
when(mock.getResultFuture()).thenReturn(resultFuture);
+ when(mock.closeAsync()).thenReturn(CompletableFuture.completedFuture(null));
return mock;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2c40697e/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java
index ba72293..0c238ce 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java
@@ -152,7 +152,7 @@ public class JobManagerRunnerTest extends TestLogger {
assertThat(resultFuture.get(), is(archivedExecutionGraph));
} finally {
- jobManagerRunner.shutdown();
+ jobManagerRunner.close();
}
}
@@ -176,7 +176,7 @@ public class JobManagerRunnerTest extends TestLogger {
assertThat(ExceptionUtils.stripExecutionException(ee), instanceOf(JobNotFinishedException.class));
}
} finally {
- jobManagerRunner.shutdown();
+ jobManagerRunner.close();
}
}
@@ -191,7 +191,7 @@ public class JobManagerRunnerTest extends TestLogger {
assertThat(resultFuture.isDone(), is(false));
- jobManagerRunner.shutdown();
+ jobManagerRunner.closeAsync();
try {
resultFuture.get();
@@ -200,7 +200,7 @@ public class JobManagerRunnerTest extends TestLogger {
assertThat(ExceptionUtils.stripExecutionException(ee), instanceOf(JobNotFinishedException.class));
}
} finally {
- jobManagerRunner.shutdown();
+ jobManagerRunner.close();
}
}
[2/3] flink git commit: [FLINK-8774] [flip6] Make shut down of
ResourceManagerRunner non blocking
Posted by tr...@apache.org.
[FLINK-8774] [flip6] Make shut down of ResourceManagerRunner non blocking
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c9787b69
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c9787b69
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c9787b69
Branch: refs/heads/master
Commit: c9787b6992f05b429edb0678050a1273900d1a63
Parents: 2c40697
Author: Till Rohrmann <tr...@apache.org>
Authored: Sat Feb 24 16:24:25 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Sat Feb 24 23:19:33 2018 +0100
----------------------------------------------------------------------
.../flink/runtime/minicluster/MiniCluster.java | 2 +-
.../resourcemanager/ResourceManagerRunner.java | 32 ++++++++------------
2 files changed, 14 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c9787b69/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
index 01be01d..5b1fddb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
@@ -437,7 +437,7 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
if (resourceManagerRunner != null) {
try {
- resourceManagerRunner.shutDown();
+ resourceManagerRunner.close();
} catch (Throwable t) {
exception = ExceptionUtils.firstOrSuppressed(t, exception);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/c9787b69/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
index 26316ec..9daf96e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
@@ -20,26 +20,26 @@ package org.apache.flink.runtime.resourcemanager;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.FutureUtils;
import org.apache.flink.runtime.entrypoint.ClusterInformation;
import org.apache.flink.runtime.heartbeat.HeartbeatServices;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.AutoCloseableAsync;
import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
/**
* Simple {@link StandaloneResourceManager} runner. It instantiates the resource manager's services
* and handles fatal errors by shutting the resource manager down.
*/
-public class ResourceManagerRunner implements FatalErrorHandler {
+public class ResourceManagerRunner implements FatalErrorHandler, AutoCloseableAsync {
private static final Logger LOG = LoggerFactory.getLogger(ResourceManagerRunner.class);
@@ -101,24 +101,18 @@ public class ResourceManagerRunner implements FatalErrorHandler {
resourceManager.start();
}
- public void shutDown() throws Exception {
- // wait for the completion
- shutDownInternally().get();
- }
-
- private CompletableFuture<Void> shutDownInternally() {
+ @Override
+ public CompletableFuture<Void> closeAsync() {
synchronized (lock) {
resourceManager.shutDown();
- return resourceManager.getTerminationFuture()
- .thenAccept(
- ignored -> {
- try {
- resourceManagerRuntimeServices.shutDown();
- } catch (Exception e) {
- throw new CompletionException(new FlinkException("Could not properly shut down the resource manager runtime services.", e));
- }
- });
+ return FutureUtils.runAfterwards(
+ resourceManager.getTerminationFuture(),
+ () -> {
+ synchronized (lock) {
+ resourceManagerRuntimeServices.shutDown();
+ }
+ });
}
}
@@ -130,7 +124,7 @@ public class ResourceManagerRunner implements FatalErrorHandler {
public void onFatalError(Throwable exception) {
LOG.error("Encountered fatal error.", exception);
- CompletableFuture<Void> shutdownFuture = shutDownInternally();
+ CompletableFuture<Void> shutdownFuture = closeAsync();
shutdownFuture.whenComplete(
(Void ignored, Throwable throwable) -> {
[3/3] flink git commit: [FLINK-8775] [flip6] Non blocking MiniCluster
shut down
Posted by tr...@apache.org.
[FLINK-8775] [flip6] Non blocking MiniCluster shut down
This closes #5576.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4e7f03e4
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4e7f03e4
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4e7f03e4
Branch: refs/heads/master
Commit: 4e7f03e418ced740a731c9323dabef7a594db43c
Parents: c9787b6
Author: Till Rohrmann <tr...@apache.org>
Authored: Sat Feb 24 16:20:58 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Sat Feb 24 23:44:53 2018 +0100
----------------------------------------------------------------------
.../flink/runtime/dispatcher/Dispatcher.java | 25 +-
.../flink/runtime/minicluster/MiniCluster.java | 334 +++++++++++--------
.../runtime/minicluster/MiniClusterITCase.java | 4 +-
.../Flip6LocalStreamEnvironment.java | 2 +-
4 files changed, 215 insertions(+), 150 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/4e7f03e4/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
index b31d04d..bb60277 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
@@ -254,16 +254,22 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
jobManagerRunner.getResultFuture().whenCompleteAsync(
(ArchivedExecutionGraph archivedExecutionGraph, Throwable throwable) -> {
- if (archivedExecutionGraph != null) {
- jobReachedGloballyTerminalState(archivedExecutionGraph);
- } else {
- final Throwable strippedThrowable = ExceptionUtils.stripCompletionException(throwable);
-
- if (strippedThrowable instanceof JobNotFinishedException) {
- jobNotFinished(jobId);
+ // check if we are still the active JobManagerRunner by checking the identity
+ //noinspection ObjectEquality
+ if (jobManagerRunner == jobManagerRunners.get(jobId)) {
+ if (archivedExecutionGraph != null) {
+ jobReachedGloballyTerminalState(archivedExecutionGraph);
} else {
- onFatalError(new FlinkException("JobManagerRunner for job " + jobId + " failed.", strippedThrowable));
+ final Throwable strippedThrowable = ExceptionUtils.stripCompletionException(throwable);
+
+ if (strippedThrowable instanceof JobNotFinishedException) {
+ jobNotFinished(jobId);
+ } else {
+ onFatalError(new FlinkException("JobManagerRunner for job " + jobId + " failed.", strippedThrowable));
+ }
}
+ } else {
+ log.debug("There is a newer JobManagerRunner for the job {}.", jobId);
}
}, getMainThreadExecutor());
@@ -294,6 +300,9 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
@Override
public CompletableFuture<Collection<JobID>> listJobs(Time timeout) {
+ if (jobManagerRunners.isEmpty()) {
+ System.out.println("empty");
+ }
return CompletableFuture.completedFuture(
Collections.unmodifiableSet(new HashSet<>(jobManagerRunners.keySet())));
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4e7f03e4/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
index 5b1fddb..3bd17d1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
@@ -69,14 +69,18 @@ import com.typesafe.config.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nonnull;
import javax.annotation.concurrent.GuardedBy;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
import java.util.concurrent.ExecutionException;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -97,6 +101,8 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
private final Time rpcTimeout;
+ private CompletableFuture<Void> terminationFuture;
+
@GuardedBy("lock")
private MetricRegistryImpl metricRegistry;
@@ -161,6 +167,7 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
this.miniClusterConfiguration = checkNotNull(miniClusterConfiguration, "config may not be null");
this.rpcTimeout = Time.seconds(10L);
+ this.terminationFuture = CompletableFuture.completedFuture(null);
running = false;
}
@@ -195,6 +202,9 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
LOG.info("Starting Flink Mini Cluster");
LOG.debug("Using configuration {}", miniClusterConfiguration);
+ // create a new termination future
+ terminationFuture = new CompletableFuture<>();
+
final Configuration configuration = miniClusterConfiguration.getConfiguration();
final Time rpcTimeout = miniClusterConfiguration.getRpcTimeout();
final int numTaskManagers = miniClusterConfiguration.getNumTaskManagers();
@@ -349,7 +359,7 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
catch (Exception e) {
// cleanup everything
try {
- shutdownInternally();
+ close();
} catch (Exception ee) {
e.addSuppressed(ee);
}
@@ -370,141 +380,70 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
* <p>This method shuts down all started services and components,
* even if an exception occurs in the process of shutting down some component.
*
- * @throws Exception Thrown, if the shutdown did not complete cleanly.
+ * @return Future which is completed once the MiniCluster has been completely shut down
*/
- public void shutdown() throws Exception {
+ @Override
+ public CompletableFuture<Void> closeAsync() {
synchronized (lock) {
if (running) {
LOG.info("Shutting down Flink Mini Cluster");
try {
- shutdownInternally();
- } finally {
- running = false;
- }
- LOG.info("Flink Mini Cluster is shut down");
- }
- }
- }
-
- @GuardedBy("lock")
- private void shutdownInternally() throws Exception {
- // this should always be called under the lock
- assert Thread.holdsLock(lock);
-
- // collect the first exception, but continue and add all successive
- // exceptions as suppressed
- Throwable exception = null;
+ final int numComponents = 2 + miniClusterConfiguration.getNumTaskManagers();
+ final Collection<CompletableFuture<Void>> componentTerminationFutures = new ArrayList<>(numComponents);
- // cancel all jobs and shut down the job dispatcher
- if (dispatcher != null) {
- try {
- RpcUtils.terminateRpcEndpoint(dispatcher, rpcTimeout);
- } catch (Exception e) {
- exception = e;
- }
- dispatcher = null;
- }
-
- if (dispatcherRestEndpoint != null) {
- try {
- dispatcherRestEndpoint.shutDownAsync().get();
- } catch (Exception e) {
- exception = ExceptionUtils.firstOrSuppressed(e, exception);
- }
-
- dispatcherRestEndpoint = null;
- }
-
- if (resourceManagerLeaderRetriever != null) {
- try {
- resourceManagerLeaderRetriever.stop();
- } catch (Exception e) {
- exception = ExceptionUtils.firstOrSuppressed(e, exception);
- }
-
- resourceManagerLeaderRetriever = null;
- }
+ componentTerminationFutures.add(shutDownDispatcher());
- if (dispatcherLeaderRetriever != null) {
- try {
- dispatcherLeaderRetriever.stop();
- } catch (Exception e) {
- exception = ExceptionUtils.firstOrSuppressed(e, exception);
- }
-
- dispatcherLeaderRetriever = null;
- }
-
- if (resourceManagerRunner != null) {
- try {
- resourceManagerRunner.close();
- } catch (Throwable t) {
- exception = ExceptionUtils.firstOrSuppressed(t, exception);
- }
- }
-
- if (taskManagers != null) {
- for (TaskExecutor tm : taskManagers) {
- if (tm != null) {
- try {
- tm.shutDown();
- // wait for the TaskManager to properly terminate
- tm.getTerminationFuture().get();
- } catch (Throwable t) {
- exception = ExceptionUtils.firstOrSuppressed(t, exception);
+ if (resourceManagerRunner != null) {
+ componentTerminationFutures.add(resourceManagerRunner.closeAsync());
+ resourceManagerRunner = null;
}
- }
- }
- taskManagers = null;
- }
- // metrics shutdown
- if (metricRegistry != null) {
- metricRegistry.shutdown();
- metricRegistry = null;
- }
-
- // shut down the RpcServices
- exception = shutDownRpc(commonRpcService, exception);
- exception = shutDownRpc(jobManagerRpcService, exception);
- exception = shutDownRpcs(taskManagerRpcServices, exception);
- exception = shutDownRpc(resourceManagerRpcService, exception);
- commonRpcService = null;
- jobManagerRpcService = null;
- taskManagerRpcServices = null;
- resourceManagerRpcService = null;
-
- if (blobCacheService != null) {
- try {
- blobCacheService.close();
- } catch (Exception e) {
- exception = ExceptionUtils.firstOrSuppressed(e, exception);
- }
- blobCacheService = null;
- }
- // shut down the blob server
- if (blobServer != null) {
- try {
- blobServer.close();
- } catch (Exception e) {
- exception = ExceptionUtils.firstOrSuppressed(e, exception);
- }
- blobServer = null;
- }
+ if (taskManagers != null) {
+ for (TaskExecutor tm : taskManagers) {
+ if (tm != null) {
+ tm.shutDown();
+ componentTerminationFutures.add(tm.getTerminationFuture());
+ }
+ }
+ taskManagers = null;
+ }
- // shut down high-availability services
- if (haServices != null) {
- try {
- haServices.closeAndCleanupAllData();
- } catch (Exception e) {
- exception = ExceptionUtils.firstOrSuppressed(e, exception);
+ final FutureUtils.ConjunctFuture<Void> componentsTerminationFuture = FutureUtils.completeAll(componentTerminationFutures);
+
+ final CompletableFuture<Void> metricRegistryTerminationFuture = FutureUtils.runAfterwards(
+ componentsTerminationFuture,
+ () -> {
+ synchronized (lock) {
+ // metrics shutdown
+ if (metricRegistry != null) {
+ metricRegistry.shutdown();
+ metricRegistry = null;
+ }
+ }
+ });
+
+ // shut down the RpcServices
+ final CompletableFuture<Void> rpcServicesTerminationFuture = metricRegistryTerminationFuture
+ .thenCompose((Void ignored) -> terminateRpcServices());
+
+ final CompletableFuture<Void> remainingServicesTerminationFuture = FutureUtils.runAfterwards(
+ rpcServicesTerminationFuture,
+ this::terminateMiniClusterServices);
+
+ remainingServicesTerminationFuture.whenComplete(
+ (Void ignored, Throwable throwable) -> {
+ if (throwable != null) {
+ terminationFuture.completeExceptionally(ExceptionUtils.stripCompletionException(throwable));
+ } else {
+ terminationFuture.complete(null);
+ }
+ });
+ } finally {
+ running = false;
+ }
}
- haServices = null;
- }
- // if anything went wrong, throw the first error with all the additional suppressed exceptions
- if (exception != null) {
- ExceptionUtils.rethrowException(exception, "Error while shutting down mini cluster");
+ return terminationFuture;
}
}
@@ -704,6 +643,137 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
}
// ------------------------------------------------------------------------
+ // Internal methods
+ // ------------------------------------------------------------------------
+
+ @GuardedBy("lock")
+ private CompletableFuture<Void> shutDownDispatcher() {
+
+ final Collection<CompletableFuture<Void>> terminationFutures = new ArrayList<>(2);
+
+ // cancel all jobs and shut down the job dispatcher
+ if (dispatcher != null) {
+ dispatcher.shutDown();
+ terminationFutures.add(dispatcher.getTerminationFuture());
+
+ dispatcher = null;
+ }
+
+ if (dispatcherRestEndpoint != null) {
+ terminationFutures.add(dispatcherRestEndpoint.shutDownAsync());
+
+ dispatcherRestEndpoint = null;
+ }
+
+ final FutureUtils.ConjunctFuture<Void> dispatcherTerminationFuture = FutureUtils.completeAll(terminationFutures);
+
+ return FutureUtils.runAfterwards(
+ dispatcherTerminationFuture,
+ () -> {
+ Exception exception = null;
+
+ synchronized (lock) {
+ if (resourceManagerLeaderRetriever != null) {
+ try {
+ resourceManagerLeaderRetriever.stop();
+ } catch (Exception e) {
+ exception = ExceptionUtils.firstOrSuppressed(e, exception);
+ }
+
+ resourceManagerLeaderRetriever = null;
+ }
+
+ if (dispatcherLeaderRetriever != null) {
+ try {
+ dispatcherLeaderRetriever.stop();
+ } catch (Exception e) {
+ exception = ExceptionUtils.firstOrSuppressed(e, exception);
+ }
+
+ dispatcherLeaderRetriever = null;
+ }
+ }
+
+ if (exception != null) {
+ throw exception;
+ }
+ });
+ }
+
+ private void terminateMiniClusterServices() throws Exception {
+ // collect the first exception, but continue and add all successive
+ // exceptions as suppressed
+ Exception exception = null;
+
+ synchronized (lock) {
+ if (blobCacheService != null) {
+ try {
+ blobCacheService.close();
+ } catch (Exception e) {
+ exception = ExceptionUtils.firstOrSuppressed(e, exception);
+ }
+ blobCacheService = null;
+ }
+
+ // shut down the blob server
+ if (blobServer != null) {
+ try {
+ blobServer.close();
+ } catch (Exception e) {
+ exception = ExceptionUtils.firstOrSuppressed(e, exception);
+ }
+ blobServer = null;
+ }
+
+ // shut down high-availability services
+ if (haServices != null) {
+ try {
+ haServices.closeAndCleanupAllData();
+ } catch (Exception e) {
+ exception = ExceptionUtils.firstOrSuppressed(e, exception);
+ }
+ haServices = null;
+ }
+
+ if (exception != null) {
+ throw exception;
+ }
+ }
+ }
+
+ @Nonnull
+ private CompletionStage<Void> terminateRpcServices() {
+ final int numRpcServices;
+ if (miniClusterConfiguration.getRpcServiceSharing() == MiniClusterConfiguration.RpcServiceSharing.SHARED) {
+ numRpcServices = 1;
+ } else {
+ numRpcServices = 1 + 2 + miniClusterConfiguration.getNumTaskManagers(); // common, JM, RM, TMs
+ }
+
+ final Collection<CompletableFuture<?>> rpcTerminationFutures = new ArrayList<>(numRpcServices);
+
+ synchronized (lock) {
+ rpcTerminationFutures.add(commonRpcService.stopService());
+
+ if (miniClusterConfiguration.getRpcServiceSharing() != MiniClusterConfiguration.RpcServiceSharing.SHARED) {
+ rpcTerminationFutures.add(jobManagerRpcService.stopService());
+ rpcTerminationFutures.add(resourceManagerRpcService.stopService());
+
+ for (RpcService taskManagerRpcService : taskManagerRpcServices) {
+ rpcTerminationFutures.add(taskManagerRpcService.stopService());
+ }
+ }
+
+ commonRpcService = null;
+ jobManagerRpcService = null;
+ taskManagerRpcServices = null;
+ resourceManagerRpcService = null;
+ }
+
+ return FutureUtils.completeAll(rpcTerminationFutures);
+ }
+
+ // ------------------------------------------------------------------------
// miscellaneous utilities
// ------------------------------------------------------------------------
@@ -743,16 +813,6 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
return priorException;
}
- @Override
- public CompletableFuture<Void> closeAsync() {
- try {
- shutdown();
- return CompletableFuture.completedFuture(null);
- } catch (Exception e) {
- return FutureUtils.completedExceptionally(e);
- }
- }
-
private class TerminatingFatalErrorHandler implements FatalErrorHandler {
private final int index;
@@ -784,11 +844,7 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync {
@Override
public void onFatalError(Throwable exception) {
LOG.warn("Error in MiniCluster. Shutting the MiniCluster down.", exception);
- try {
- shutdown();
- } catch (Exception e) {
- LOG.warn("Could not shut down the MiniCluster.", e);
- }
+ closeAsync();
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4e7f03e4/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
index d974998..646827a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
@@ -67,7 +67,7 @@ public class MiniClusterITCase extends TestLogger {
executeJob(miniCluster);
}
finally {
- miniCluster.shutdown();
+ miniCluster.close();
}
}
@@ -84,7 +84,7 @@ public class MiniClusterITCase extends TestLogger {
executeJob(miniCluster);
}
finally {
- miniCluster.shutdown();
+ miniCluster.close();
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4e7f03e4/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java
index 5941ee5..4cc23fc 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/Flip6LocalStreamEnvironment.java
@@ -105,7 +105,7 @@ public class Flip6LocalStreamEnvironment extends LocalStreamEnvironment {
}
finally {
transformations.clear();
- miniCluster.shutdown();
+ miniCluster.close();
}
}
}