You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2017/09/19 22:34:42 UTC
[2/2] flink git commit: [FLINK-7528] Create DispatcherRestEndpoint
and integrate with Dispatcher
[FLINK-7528] Create DispatcherRestEndpoint and integrate with Dispatcher
This commit creates the DispatcherRestEndpoint and integrates it with the
Dispatcher. The DispatcherRestEndpoint is created in the SessionClusterEntrypoint
and its address is passed to the Dispatcher such that it can answer the
requestRestAddress RPC.
This closes #4598.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/6a62f145
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/6a62f145
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/6a62f145
Branch: refs/heads/master
Commit: 6a62f1455313ee8fae0ff79945da61fb67ec8edb
Parents: 75e84e0
Author: Till Rohrmann <tr...@apache.org>
Authored: Thu Aug 17 15:25:33 2017 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Wed Sep 20 00:34:13 2017 +0200
----------------------------------------------------------------------
.../MesosApplicationMasterRunner.java | 36 ++--
.../clusterframework/MesosJobManager.scala | 6 +-
.../runtime/webmonitor/WebRuntimeMonitor.java | 26 +--
.../webmonitor/utils/WebFrontendBootstrap.java | 23 ++-
.../runtime/webmonitor/RedirectHandlerTest.java | 10 +-
.../webmonitor/WebRuntimeMonitorITCase.java | 107 +++++------
.../handlers/HandlerRedirectUtilsTest.java | 32 ++--
.../webmonitor/metrics/MetricFetcherTest.java | 1 -
.../runtime/akka/AkkaJobManagerGateway.java | 17 +-
.../clusterframework/BootstrapTools.java | 4 +-
.../flink/runtime/dispatcher/Dispatcher.java | 15 +-
.../runtime/dispatcher/DispatcherGateway.java | 3 +-
.../dispatcher/DispatcherRestEndpoint.java | 41 +++++
.../dispatcher/StandaloneDispatcher.java | 8 +-
.../entrypoint/SessionClusterEntrypoint.java | 21 ++-
.../runtime/jobmaster/JobManagerGateway.java | 7 -
.../minicluster/StandaloneMiniCluster.java | 1 +
.../flink/runtime/rest/RestServerEndpoint.java | 176 ++++++++++++-------
.../runtime/rest/handler/RedirectHandler.java | 64 ++++---
.../rest/handler/util/HandlerRedirectUtils.java | 26 ++-
.../flink/runtime/webmonitor/WebMonitor.java | 9 +-
.../ContaineredJobManager.scala | 6 +-
.../flink/runtime/jobmanager/JobManager.scala | 51 +++---
.../runtime/messages/JobManagerMessages.scala | 15 +-
.../runtime/minicluster/FlinkMiniCluster.scala | 22 ++-
.../minicluster/LocalFlinkMiniCluster.scala | 15 +-
.../runtime/dispatcher/DispatcherTest.java | 4 +-
.../jobmanager/JobManagerHARecoveryTest.java | 6 +-
.../runtime/jobmanager/JobManagerTest.java | 5 +
.../flink/runtime/jobmanager/JobSubmitTest.java | 3 +
.../JobManagerLeaderElectionTest.java | 5 +-
.../runtime/metrics/TaskManagerMetricsTest.java | 2 +
.../flink/runtime/rest/RestEndpointITCase.java | 3 +-
...askManagerComponentsStartupShutdownTest.java | 3 +-
.../TaskManagerProcessReapingTestBase.java | 3 +
.../jobmanager/JobManagerRegistrationTest.scala | 1 +
.../runtime/testingUtils/TestingCluster.scala | 11 +-
.../testingUtils/TestingJobManager.scala | 6 +-
.../runtime/testingUtils/TestingUtils.scala | 1 +
.../apache/flink/test/util/TestBaseUtils.java | 50 ++++--
...ctTaskManagerProcessFailureRecoveryTest.java | 2 +
.../recovery/ProcessFailureCancelingITCase.java | 2 +
.../AbstractOperatorRestoreTestBase.java | 1 +
.../YARNSessionCapacitySchedulerITCase.java | 118 ++++++-------
.../flink/yarn/TestingYarnJobManager.scala | 6 +-
.../flink/yarn/YarnApplicationMasterRunner.java | 40 ++---
.../org/apache/flink/yarn/YarnJobManager.scala | 6 +-
47 files changed, 586 insertions(+), 434 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
index 591b7f3..0ebaa2e 100755
--- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
+++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
@@ -282,22 +282,7 @@ public class MesosApplicationMasterRunner {
ioExecutor,
HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
- // 1: the JobManager
- LOG.debug("Starting JobManager actor");
-
- // we start the JobManager with its standard name
- ActorRef jobManager = JobManager.startJobManagerActors(
- config,
- actorSystem,
- futureExecutor,
- ioExecutor,
- highAvailabilityServices,
- Option.apply(JobMaster.JOB_MANAGER_NAME),
- Option.apply(JobMaster.ARCHIVE_NAME),
- getJobManagerClass(),
- getArchivistClass())._1();
-
- // 2: the web monitor
+ // 1: the web monitor
LOG.debug("Starting Web Frontend");
Time webMonitorTimeout = Time.milliseconds(config.getLong(WebOptions.TIMEOUT));
@@ -309,13 +294,28 @@ public class MesosApplicationMasterRunner {
new AkkaQueryServiceRetriever(actorSystem, webMonitorTimeout),
webMonitorTimeout,
futureExecutor,
- AkkaUtils.getAkkaURL(actorSystem, jobManager),
LOG);
if (webMonitor != null) {
- final URL webMonitorURL = new URL("http", appMasterHostname, webMonitor.getServerPort(), "/");
+ final URL webMonitorURL = new URL(webMonitor.getRestAddress());
mesosConfig.frameworkInfo().setWebuiUrl(webMonitorURL.toExternalForm());
}
+ // 2: the JobManager
+ LOG.debug("Starting JobManager actor");
+
+ // we start the JobManager with its standard name
+ ActorRef jobManager = JobManager.startJobManagerActors(
+ config,
+ actorSystem,
+ futureExecutor,
+ ioExecutor,
+ highAvailabilityServices,
+ webMonitor != null ? Option.apply(webMonitor.getRestAddress()) : Option.empty(),
+ Option.apply(JobMaster.JOB_MANAGER_NAME),
+ Option.apply(JobMaster.ARCHIVE_NAME),
+ getJobManagerClass(),
+ getArchivistClass())._1();
+
// 3: Flink's Mesos ResourceManager
LOG.debug("Starting Mesos Flink Resource Manager");
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosJobManager.scala
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosJobManager.scala b/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosJobManager.scala
index f854a1e..9ad8eb2 100644
--- a/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosJobManager.scala
+++ b/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosJobManager.scala
@@ -66,7 +66,8 @@ class MesosJobManager(
submittedJobGraphs : SubmittedJobGraphStore,
checkpointRecoveryFactory : CheckpointRecoveryFactory,
jobRecoveryTimeout: FiniteDuration,
- metricsRegistry: Option[FlinkMetricRegistry])
+ metricsRegistry: Option[FlinkMetricRegistry],
+ optRestAddress: Option[String])
extends ContaineredJobManager(
flinkConfiguration,
futureExecutor,
@@ -82,7 +83,8 @@ class MesosJobManager(
submittedJobGraphs,
checkpointRecoveryFactory,
jobRecoveryTimeout,
- metricsRegistry) {
+ metricsRegistry,
+ optRestAddress) {
val jobPollingInterval: FiniteDuration = 5 seconds
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
index 9dbbf2f..a15c312 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
@@ -123,7 +123,7 @@ public class WebRuntimeMonitor implements WebMonitor {
private final SSLContext serverSSLContext;
- private final CompletableFuture<String> jobManagerAddressFuture = new CompletableFuture<>();
+ private final CompletableFuture<String> localRestAddress = new CompletableFuture<>();
private final Time timeout;
@@ -273,7 +273,7 @@ public class WebRuntimeMonitor implements WebMonitor {
new TaskManagerLogHandler(
retriever,
executor,
- jobManagerAddressFuture,
+ localRestAddress,
timeout,
TaskManagerLogHandler.FileMode.LOG,
config,
@@ -283,7 +283,7 @@ public class WebRuntimeMonitor implements WebMonitor {
new TaskManagerLogHandler(
retriever,
executor,
- jobManagerAddressFuture,
+ localRestAddress,
timeout,
TaskManagerLogHandler.FileMode.STDOUT,
config,
@@ -296,13 +296,13 @@ public class WebRuntimeMonitor implements WebMonitor {
.GET("/jobmanager/log", logFiles.logFile == null ? new ConstantTextHandler("(log file unavailable)") :
new StaticFileServerHandler<>(
retriever,
- jobManagerAddressFuture,
+ localRestAddress,
timeout,
logFiles.logFile,
enableSSL))
.GET("/jobmanager/stdout", logFiles.stdOutFile == null ? new ConstantTextHandler("(stdout file unavailable)") :
- new StaticFileServerHandler<>(retriever, jobManagerAddressFuture, timeout, logFiles.stdOutFile,
+ new StaticFileServerHandler<>(retriever, localRestAddress, timeout, logFiles.stdOutFile,
enableSSL));
get(router, new JobManagerMetricsHandler(executor, metricFetcher));
@@ -355,7 +355,7 @@ public class WebRuntimeMonitor implements WebMonitor {
// this handler serves all the static contents
router.GET("/:*", new StaticFileServerHandler<>(
retriever,
- jobManagerAddressFuture,
+ localRestAddress,
timeout,
webRootDir,
enableSSL));
@@ -377,6 +377,8 @@ public class WebRuntimeMonitor implements WebMonitor {
}
this.netty = new WebFrontendBootstrap(router, LOG, uploadDir, serverSSLContext, configuredAddress, configuredPort, config);
+
+ localRestAddress.complete(netty.getRestAddress());
}
/**
@@ -420,11 +422,8 @@ public class WebRuntimeMonitor implements WebMonitor {
}
@Override
- public void start(String jobManagerAkkaUrl) throws Exception {
- LOG.info("Starting with JobManager {} on port {}", jobManagerAkkaUrl, getServerPort());
-
+ public void start() throws Exception {
synchronized (startupShutdownLock) {
- jobManagerAddressFuture.complete(jobManagerAkkaUrl);
leaderRetrievalService.start(retriever);
long delay = backPressureStatsTracker.getCleanUpInterval();
@@ -466,6 +465,11 @@ public class WebRuntimeMonitor implements WebMonitor {
return netty.getServerPort();
}
+ @Override
+ public String getRestAddress() {
+ return netty.getRestAddress();
+ }
+
private void cleanup() {
if (!cleanedUp.compareAndSet(false, true)) {
return;
@@ -526,7 +530,7 @@ public class WebRuntimeMonitor implements WebMonitor {
// ------------------------------------------------------------------------
private RuntimeMonitorHandler handler(RequestHandler handler) {
- return new RuntimeMonitorHandler(cfg, handler, retriever, jobManagerAddressFuture, timeout,
+ return new RuntimeMonitorHandler(cfg, handler, retriever, localRestAddress, timeout,
serverSSLContext != null);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrap.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrap.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrap.java
index 2c47478..c3f85d9 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrap.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrap.java
@@ -19,6 +19,7 @@
package org.apache.flink.runtime.webmonitor.utils;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.JobManagerOptions;
import org.apache.flink.runtime.net.SSLUtils;
import org.apache.flink.runtime.webmonitor.HttpRequestHandler;
import org.apache.flink.runtime.webmonitor.PipelineErrorHandler;
@@ -43,7 +44,9 @@ import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLEngine;
import java.io.File;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
/**
* This classes encapsulates the boot-strapping of netty for the web-frontend.
@@ -55,6 +58,7 @@ public class WebFrontendBootstrap {
private final SSLContext serverSSLContext;
private final ServerBootstrap bootstrap;
private final Channel serverChannel;
+ private final String restAddress;
public WebFrontendBootstrap(
Router router,
@@ -63,7 +67,7 @@ public class WebFrontendBootstrap {
SSLContext sslContext,
String configuredAddress,
int configuredPort,
- final Configuration config) throws InterruptedException {
+ final Configuration config) throws InterruptedException, UnknownHostException {
this.router = Preconditions.checkNotNull(router);
this.log = Preconditions.checkNotNull(log);
this.uploadDir = directory;
@@ -110,10 +114,21 @@ public class WebFrontendBootstrap {
this.serverChannel = ch.sync().channel();
InetSocketAddress bindAddress = (InetSocketAddress) serverChannel.localAddress();
- String address = bindAddress.getAddress().getHostAddress();
+
+ InetAddress inetAddress = bindAddress.getAddress();
+ final String address;
+
+ if (inetAddress.isAnyLocalAddress()) {
+ address = config.getString(JobManagerOptions.ADDRESS, InetAddress.getLocalHost().getHostName());
+ } else {
+ address = inetAddress.getHostAddress();
+ }
+
int port = bindAddress.getPort();
this.log.info("Web frontend listening at {}" + ':' + "{}", address, port);
+
+ this.restAddress = address + ':' + port;
}
public ServerBootstrap getBootstrap() {
@@ -134,6 +149,10 @@ public class WebFrontendBootstrap {
return -1;
}
+ public String getRestAddress() {
+ return restAddress;
+ }
+
public void shutdown() {
if (this.serverChannel != null) {
this.serverChannel.close().awaitUninterruptibly();
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/RedirectHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/RedirectHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/RedirectHandlerTest.java
index 2bd6673..23cad19 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/RedirectHandlerTest.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/RedirectHandlerTest.java
@@ -66,8 +66,7 @@ public class RedirectHandlerTest extends TestLogger {
@Test
public void testRedirectHandler() throws Exception {
final String restPath = "/testing";
- final String correctAddress = "foobar";
- final String incorrectAddres = "barfoo";
+ final String correctAddress = "foobar:21345";
final String redirectionAddress = "foobar:12345";
final String expectedRedirection = "http://" + redirectionAddress + restPath;
@@ -78,11 +77,10 @@ public class RedirectHandlerTest extends TestLogger {
final GatewayRetriever<RestfulGateway> gatewayRetriever = mock(GatewayRetriever.class);
final RestfulGateway redirectionGateway = mock(RestfulGateway.class);
- when(redirectionGateway.getAddress()).thenReturn(incorrectAddres);
when(redirectionGateway.requestRestAddress(any(Time.class))).thenReturn(CompletableFuture.completedFuture(redirectionAddress));
final RestfulGateway localGateway = mock(RestfulGateway.class);
- when(localGateway.getAddress()).thenReturn(correctAddress);
+ when(localGateway.requestRestAddress(any(Time.class))).thenReturn(CompletableFuture.completedFuture(correctAddress));
when(gatewayRetriever.getNow()).thenReturn(Optional.empty(), Optional.of(redirectionGateway), Optional.of(localGateway));
@@ -103,12 +101,12 @@ public class RedirectHandlerTest extends TestLogger {
configuration);
try (HttpTestClient httpClient = new HttpTestClient("localhost", bootstrap.getServerPort())) {
- // 1. without completed local address future --> Service unavailable
+ // 1. without completed local address future --> Internal server error
httpClient.sendGetRequest(restPath, FutureUtils.toFiniteDuration(timeout));
HttpTestClient.SimpleHttpResponse response = httpClient.getNextResponse(FutureUtils.toFiniteDuration(timeout));
- Assert.assertEquals(HttpResponseStatus.SERVICE_UNAVAILABLE, response.getStatus());
+ Assert.assertEquals(HttpResponseStatus.INTERNAL_SERVER_ERROR, response.getStatus());
// 2. with completed local address future but no leader gateway available --> Service unavailable
localAddressFuture.complete(correctAddress);
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java
index 72498ee..e67c5ce 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java
@@ -59,11 +59,13 @@ import java.util.Scanner;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import scala.Option;
import scala.Some;
import scala.Tuple2;
import scala.concurrent.duration.Deadline;
import scala.concurrent.duration.FiniteDuration;
+import static org.apache.flink.configuration.ConfigConstants.LOCAL_START_WEBSERVER;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -91,13 +93,19 @@ public class WebRuntimeMonitorITCase extends TestLogger {
final Deadline deadline = TEST_TIMEOUT.fromNow();
TestingCluster flink = null;
- WebRuntimeMonitor webMonitor = null;
+
+ final Configuration configuration = new Configuration();
+
+ // activate the web monitor
+ configuration.setBoolean(LOCAL_START_WEBSERVER, true);
+ configuration.setInteger(WebOptions.PORT, 0);
try {
- // Flink w/o a web monitor
- flink = new TestingCluster(new Configuration());
+ // Flink with a web monitor
+ flink = new TestingCluster(configuration);
flink.start(true);
- webMonitor = startWebRuntimeMonitor(flink, TIMEOUT);
+
+ WebMonitor webMonitor = flink.webMonitor().get();
try (HttpTestClient client = new HttpTestClient("localhost", webMonitor.getServerPort())) {
String expected = new Scanner(new File(mainResourcesPath + "/index.html"))
@@ -125,10 +133,6 @@ public class WebRuntimeMonitorITCase extends TestLogger {
if (flink != null) {
flink.shutdown();
}
-
- if (webMonitor != null) {
- webMonitor.stop();
- }
}
}
@@ -191,11 +195,12 @@ public class WebRuntimeMonitorITCase extends TestLogger {
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
highAvailabilityServices,
+ Option.apply(webMonitor[i].getRestAddress()),
JobManager.class,
MemoryArchivist.class)._1();
jobManagerAddress[i] = AkkaUtils.getAkkaURL(jobManagerSystem[i], jobManager[i]);
- webMonitor[i].start(jobManagerAddress[i]);
+ webMonitor[i].start();
}
LeaderRetrievalService lrs = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID);
@@ -320,7 +325,7 @@ public class WebRuntimeMonitorITCase extends TestLogger {
TIMEOUT,
TestingUtils.defaultExecutor());
- webRuntimeMonitor.start("akka://schmakka");
+ webRuntimeMonitor.start();
try (HttpTestClient client = new HttpTestClient(
"localhost", webRuntimeMonitor.getServerPort())) {
@@ -359,12 +364,18 @@ public class WebRuntimeMonitorITCase extends TestLogger {
final Deadline deadline = TEST_TIMEOUT.fromNow();
TestingCluster flink = null;
- WebRuntimeMonitor webMonitor = null;
+
+ final Configuration configuration = new Configuration();
+
+ // activate the web monitor
+ configuration.setBoolean(LOCAL_START_WEBSERVER, true);
+ configuration.setInteger(WebOptions.PORT, 0);
try {
- flink = new TestingCluster(new Configuration());
+ flink = new TestingCluster(configuration);
flink.start(true);
- webMonitor = startWebRuntimeMonitor(flink, TIMEOUT);
+
+ WebRuntimeMonitor webMonitor = (WebRuntimeMonitor) flink.webMonitor().get();
try (HttpTestClient client = new HttpTestClient("localhost", webMonitor.getServerPort())) {
String expectedIndex = new Scanner(new File(mainResourcesPath + "/index.html"))
@@ -405,10 +416,6 @@ public class WebRuntimeMonitorITCase extends TestLogger {
if (flink != null) {
flink.shutdown();
}
-
- if (webMonitor != null) {
- webMonitor.stop();
- }
}
}
@@ -421,16 +428,21 @@ public class WebRuntimeMonitorITCase extends TestLogger {
final Deadline deadline = TEST_TIMEOUT.fromNow();
TestingCluster flink = null;
- WebRuntimeMonitor webMonitor = null;
+
+ final Configuration configuration = new Configuration();
+
+ // activate the web monitor
+ configuration.setBoolean(LOCAL_START_WEBSERVER, true);
+ configuration.setInteger(WebOptions.PORT, 0);
try {
- flink = new TestingCluster(new Configuration());
+ flink = new TestingCluster(configuration);
flink.start(true);
- webMonitor = startWebRuntimeMonitor(flink, TIMEOUT);
+ WebRuntimeMonitor webMonitor = ((WebRuntimeMonitor) flink.webMonitor().get());
try (HttpTestClient client = new HttpTestClient("localhost", webMonitor.getServerPort())) {
String expectedIndex = new Scanner(new File(mainResourcesPath + "/index.html"))
- .useDelimiter("\\A").next();
+ .useDelimiter("\\A").next();
// 1) Request index.html from web server
client.sendGetRequest("index.html", deadline.timeLeft());
@@ -445,67 +457,28 @@ public class WebRuntimeMonitorITCase extends TestLogger {
response = client.getNextResponse(deadline.timeLeft());
assertEquals(
- "Returned status code " + response.getStatus() + " for file outside of web root.",
- HttpResponseStatus.NOT_FOUND,
- response.getStatus());
+ "Returned status code " + response.getStatus() + " for file outside of web root.",
+ HttpResponseStatus.NOT_FOUND,
+ response.getStatus());
assertFalse("Did not respond with the file, but still copied it from the JAR.",
- new File(webMonitor.getBaseDir(new Configuration()), "log4j-test.properties").exists());
+ new File(webMonitor.getBaseDir(new Configuration()), "log4j-test.properties").exists());
// 3) Request non-existing file
client.sendGetRequest("not-existing-resource", deadline.timeLeft());
response = client.getNextResponse(deadline.timeLeft());
assertEquals(
- "Unexpected status code " + response.getStatus() + " for file outside of web root.",
- HttpResponseStatus.NOT_FOUND,
- response.getStatus());
+ "Unexpected status code " + response.getStatus() + " for file outside of web root.",
+ HttpResponseStatus.NOT_FOUND,
+ response.getStatus());
}
} finally {
if (flink != null) {
flink.shutdown();
}
-
- if (webMonitor != null) {
- webMonitor.stop();
- }
}
}
- private WebRuntimeMonitor startWebRuntimeMonitor(
- TestingCluster flink,
- Time timeout) throws Exception {
-
- ActorSystem jmActorSystem = flink.jobManagerActorSystems().get().head();
- ActorRef jmActor = flink.jobManagerActors().get().head();
-
- // Needs to match the leader address from the leader retrieval service
- String jobManagerAddress = AkkaUtils.getAkkaURL(jmActorSystem, jmActor);
-
- File logDir = temporaryFolder.newFolder("log");
- Path logFile = Files.createFile(new File(logDir, "jobmanager.log").toPath());
- Files.createFile(new File(logDir, "jobmanager.out").toPath());
-
- // Web frontend on random port
- Configuration config = new Configuration();
- config.setInteger(WebOptions.PORT, 0);
- config.setString(WebOptions.LOG_PATH, logFile.toString());
-
- HighAvailabilityServices highAvailabilityServices = flink.highAvailabilityServices();
-
- WebRuntimeMonitor webMonitor = new WebRuntimeMonitor(
- config,
- highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
- highAvailabilityServices.createBlobStore(),
- new AkkaJobManagerRetriever(jmActorSystem, timeout, 0, Time.milliseconds(50L)),
- new AkkaQueryServiceRetriever(jmActorSystem, timeout),
- timeout,
- TestingUtils.defaultExecutor());
-
- webMonitor.start(jobManagerAddress);
- flink.waitForActorsToBeAlive();
- return webMonitor;
- }
-
// ------------------------------------------------------------------------
private void waitForLeaderNotification(
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/HandlerRedirectUtilsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/HandlerRedirectUtilsTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/HandlerRedirectUtilsTest.java
index 81553e4..e79be96 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/HandlerRedirectUtilsTest.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/HandlerRedirectUtilsTest.java
@@ -39,40 +39,36 @@ import static org.mockito.Mockito.when;
*/
public class HandlerRedirectUtilsTest extends TestLogger {
- private static final String localJobManagerAddress = "akka.tcp://flink@127.0.0.1:1234/user/foobar";
- private static final String remoteHostname = "127.0.0.2";
- private static final int webPort = 1235;
- private static final String remoteURL = remoteHostname + ':' + webPort;
- private static final String remotePath = "akka.tcp://flink@" + remoteURL + "/user/jobmanager";
+ private static final String localRestAddress = "http://127.0.0.1:1234";
+ private static final String remoteRestAddress = "http://127.0.0.2:1234";
@Test
- public void testGetRedirectAddressWithLocalAkkaPath() throws Exception {
+ public void testGetRedirectAddressWithLocalEqualsRemoteRESTAddress() throws Exception {
JobManagerGateway jobManagerGateway = mock(JobManagerGateway.class);
- when(jobManagerGateway.getAddress()).thenReturn("akka://flink/user/foobar");
+ when(jobManagerGateway.requestRestAddress(any(Time.class))).thenReturn(CompletableFuture.completedFuture(localRestAddress));
- Optional<CompletableFuture<String>> redirectingAddress = HandlerRedirectUtils.getRedirectAddress(
- localJobManagerAddress,
+ CompletableFuture<Optional<String>> redirectingAddressFuture = HandlerRedirectUtils.getRedirectAddress(
+ localRestAddress,
jobManagerGateway,
Time.seconds(3L));
- Assert.assertFalse(redirectingAddress.isPresent());
+ Assert.assertTrue(redirectingAddressFuture.isDone());
+ // no redirection needed
+ Assert.assertFalse(redirectingAddressFuture.get().isPresent());
}
@Test
public void testGetRedirectAddressWithRemoteAkkaPath() throws Exception {
JobManagerGateway jobManagerGateway = mock(AkkaJobManagerGateway.class);
- when(jobManagerGateway.getAddress()).thenReturn(remotePath);
- when(jobManagerGateway.getHostname()).thenReturn(remoteHostname);
- when(jobManagerGateway.requestWebPort(any(Time.class))).thenReturn(CompletableFuture.completedFuture(webPort));
- when(jobManagerGateway.requestRestAddress(any(Time.class))).thenCallRealMethod();
+ when(jobManagerGateway.requestRestAddress(any(Time.class))).thenReturn(CompletableFuture.completedFuture(remoteRestAddress));
- Optional<CompletableFuture<String>> optRedirectingAddress = HandlerRedirectUtils.getRedirectAddress(
- localJobManagerAddress,
+ CompletableFuture<Optional<String>> optRedirectingAddress = HandlerRedirectUtils.getRedirectAddress(
+ localRestAddress,
jobManagerGateway,
Time.seconds(3L));
- Assert.assertTrue(optRedirectingAddress.isPresent());
+ Assert.assertTrue(optRedirectingAddress.isDone());
- Assert.assertEquals(remoteURL, optRedirectingAddress.get().get());
+ Assert.assertEquals(remoteRestAddress, optRedirectingAddress.get().get());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/metrics/MetricFetcherTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/metrics/MetricFetcherTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/metrics/MetricFetcherTest.java
index 20c6373..09a0829 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/metrics/MetricFetcherTest.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/metrics/MetricFetcherTest.java
@@ -95,7 +95,6 @@ public class MetricFetcherTest extends TestLogger {
when(jobManagerGateway.requestTaskManagerInstances(any(Time.class)))
.thenReturn(CompletableFuture.completedFuture(Collections.singleton(taskManager)));
when(jobManagerGateway.getAddress()).thenReturn("/jm/address");
- when(jobManagerGateway.requestWebPort(any(Time.class))).thenReturn(CompletableFuture.completedFuture(0));
GatewayRetriever<JobManagerGateway> retriever = mock(AkkaJobManagerRetriever.class);
when(retriever.getNow())
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/java/org/apache/flink/runtime/akka/AkkaJobManagerGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/akka/AkkaJobManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/akka/AkkaJobManagerGateway.java
index b9a7a8d..23ba6b3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/akka/AkkaJobManagerGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/akka/AkkaJobManagerGateway.java
@@ -80,17 +80,6 @@ public class AkkaJobManagerGateway implements JobManagerGateway {
.mapTo(ClassTag$.MODULE$.apply(Integer.class)));
}
- @Override
- public CompletableFuture<Integer> requestWebPort(Time timeout) {
- CompletableFuture<JobManagerMessages.ResponseWebMonitorPort> portResponseFuture = FutureUtils.toJava(
- jobManagerGateway
- .ask(JobManagerMessages.getRequestWebMonitorPort(), FutureUtils.toFiniteDuration(timeout))
- .mapTo(ClassTag$.MODULE$.apply(JobManagerMessages.ResponseWebMonitorPort.class)));
-
- return portResponseFuture.thenApply(
- JobManagerMessages.ResponseWebMonitorPort::port);
- }
-
//--------------------------------------------------------------------------------
// Job control
//--------------------------------------------------------------------------------
@@ -270,7 +259,9 @@ public class AkkaJobManagerGateway implements JobManagerGateway {
@Override
public CompletableFuture<String> requestRestAddress(Time timeout) {
- return requestWebPort(timeout).thenApply(
- (Integer webPort) -> getHostname() + ':' + webPort);
+ return FutureUtils.toJava(
+ jobManagerGateway
+ .ask(JobManagerMessages.getRequestRestAddress(), FutureUtils.toFiniteDuration(timeout))
+ .mapTo(ClassTag$.MODULE$.apply(String.class)));
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java
index bd3e304..62cddb0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java
@@ -179,7 +179,6 @@ public class BootstrapTools {
* @param queryServiceRetriever to resolve a query service
* @param timeout for asynchronous operations
* @param executor to run asynchronous operations
- * @param jobManagerAddress the address of the JobManager for which the WebMonitor is started
* @param logger Logger for log output
* @return WebMonitor instance.
* @throws Exception
@@ -191,7 +190,6 @@ public class BootstrapTools {
MetricQueryServiceRetriever queryServiceRetriever,
Time timeout,
Executor executor,
- String jobManagerAddress,
Logger logger) throws Exception {
if (config.getInteger(WebOptions.PORT, 0) >= 0) {
@@ -209,7 +207,7 @@ public class BootstrapTools {
// start the web monitor
if (monitor != null) {
- monitor.start(jobManagerAddress);
+ monitor.start();
}
return monitor;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
index 8977415..521fd8b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java
@@ -51,6 +51,7 @@ import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@@ -80,6 +81,8 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
private final LeaderElectionService leaderElectionService;
+ private final CompletableFuture<String> restAddressFuture;
+
protected Dispatcher(
RpcService rpcService,
String endpointId,
@@ -88,7 +91,8 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
BlobServer blobServer,
HeartbeatServices heartbeatServices,
MetricRegistry metricRegistry,
- FatalErrorHandler fatalErrorHandler) throws Exception {
+ FatalErrorHandler fatalErrorHandler,
+ Optional<String> restAddress) throws Exception {
super(rpcService, endpointId, DispatcherId.generate());
this.configuration = Preconditions.checkNotNull(configuration);
@@ -106,6 +110,10 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
jobManagerRunners = new HashMap<>(16);
leaderElectionService = highAvailabilityServices.getDispatcherLeaderElectionService();
+ this.restAddressFuture = restAddress
+ .map(CompletableFuture::completedFuture)
+ .orElse(FutureUtils.completedExceptionally(new DispatcherException("The Dispatcher has not been started with a REST endpoint.")));
+
}
//------------------------------------------------------
@@ -230,6 +238,11 @@ public abstract class Dispatcher extends FencedRpcEndpoint<DispatcherId> impleme
return CompletableFuture.completedFuture(jobManagerRunners.keySet());
}
+ @Override
+ public CompletableFuture<String> requestRestAddress(Time timeout) {
+ return restAddressFuture;
+ }
+
/**
* Cleans up the job related data from the dispatcher. If cleanupHA is true, then
* the data will also be removed from HA.
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherGateway.java
index 09254c3..398befb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherGateway.java
@@ -24,6 +24,7 @@ import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.rpc.FencedRpcGateway;
import org.apache.flink.runtime.rpc.RpcTimeout;
+import org.apache.flink.runtime.webmonitor.RestfulGateway;
import java.util.Collection;
import java.util.concurrent.CompletableFuture;
@@ -31,7 +32,7 @@ import java.util.concurrent.CompletableFuture;
/**
* Gateway for the Dispatcher component.
*/
-public interface DispatcherGateway extends FencedRpcGateway<DispatcherId> {
+public interface DispatcherGateway extends FencedRpcGateway<DispatcherId>, RestfulGateway {
/**
* Submit a job to the dispatcher.
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java
new file mode 100644
index 0000000..63b1a4c
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.dispatcher;
+
+import org.apache.flink.runtime.rest.RestServerEndpoint;
+import org.apache.flink.runtime.rest.RestServerEndpointConfiguration;
+import org.apache.flink.runtime.rest.handler.AbstractRestHandler;
+
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * REST endpoint for the {@link Dispatcher} component.
+ */
+public class DispatcherRestEndpoint extends RestServerEndpoint {
+
+ public DispatcherRestEndpoint(RestServerEndpointConfiguration configuration) {
+ super(configuration);
+ }
+
+ @Override
+ protected Collection<AbstractRestHandler<?, ?, ?, ?>> initializeHandlers() {
+ return Collections.emptySet();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java
index d6d82b1..efd565a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java
@@ -32,6 +32,8 @@ import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcService;
+import java.util.Optional;
+
/**
* Dispatcher implementation which spawns a {@link JobMaster} for each
* submitted {@link JobGraph} within in the same process. This dispatcher
@@ -46,7 +48,8 @@ public class StandaloneDispatcher extends Dispatcher {
BlobServer blobServer,
HeartbeatServices heartbeatServices,
MetricRegistry metricRegistry,
- FatalErrorHandler fatalErrorHandler) throws Exception {
+ FatalErrorHandler fatalErrorHandler,
+ Optional<String> restAddress) throws Exception {
super(
rpcService,
endpointId,
@@ -55,7 +58,8 @@ public class StandaloneDispatcher extends Dispatcher {
blobServer,
heartbeatServices,
metricRegistry,
- fatalErrorHandler);
+ fatalErrorHandler,
+ restAddress);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java
index cea1688..0d14443 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java
@@ -22,16 +22,20 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.blob.BlobServer;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.dispatcher.Dispatcher;
+import org.apache.flink.runtime.dispatcher.DispatcherRestEndpoint;
import org.apache.flink.runtime.dispatcher.StandaloneDispatcher;
import org.apache.flink.runtime.heartbeat.HeartbeatServices;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.resourcemanager.ResourceManager;
+import org.apache.flink.runtime.rest.RestServerEndpointConfiguration;
import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FlinkException;
+import java.util.Optional;
+
/**
* Base class for session cluster entry points.
*/
@@ -41,6 +45,8 @@ public abstract class SessionClusterEntrypoint extends ClusterEntrypoint {
private Dispatcher dispatcher;
+ private DispatcherRestEndpoint dispatcherRestEndpoint;
+
public SessionClusterEntrypoint(Configuration configuration) {
super(configuration);
}
@@ -54,6 +60,12 @@ public abstract class SessionClusterEntrypoint extends ClusterEntrypoint {
HeartbeatServices heartbeatServices,
MetricRegistry metricRegistry) throws Exception {
+ dispatcherRestEndpoint = new DispatcherRestEndpoint(
+ RestServerEndpointConfiguration.fromConfiguration(configuration));
+
+ LOG.debug("Starting Dispatcher REST endpoint.");
+ dispatcherRestEndpoint.start();
+
resourceManager = createResourceManager(
configuration,
ResourceID.generate(),
@@ -70,7 +82,8 @@ public abstract class SessionClusterEntrypoint extends ClusterEntrypoint {
blobServer,
heartbeatServices,
metricRegistry,
- this);
+ this,
+ Optional.of(dispatcherRestEndpoint.getRestAddress()));
LOG.debug("Starting ResourceManager.");
resourceManager.start();
@@ -111,7 +124,8 @@ public abstract class SessionClusterEntrypoint extends ClusterEntrypoint {
BlobServer blobServer,
HeartbeatServices heartbeatServices,
MetricRegistry metricRegistry,
- FatalErrorHandler fatalErrorHandler) throws Exception {
+ FatalErrorHandler fatalErrorHandler,
+ Optional<String> restAddress) throws Exception {
// create the default dispatcher
return new StandaloneDispatcher(
@@ -122,7 +136,8 @@ public abstract class SessionClusterEntrypoint extends ClusterEntrypoint {
blobServer,
heartbeatServices,
metricRegistry,
- fatalErrorHandler);
+ fatalErrorHandler,
+ restAddress);
}
protected abstract ResourceManager<?> createResourceManager(
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerGateway.java
index f2aaf17..0ff8884 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerGateway.java
@@ -55,13 +55,6 @@ public interface JobManagerGateway extends RestfulGateway {
CompletableFuture<Integer> requestBlobServerPort(Time timeout);
/**
- * Returns the port of the web runtime monitor serving requests for the JobManager endpoint.
- *
- * @return Port of the WebRuntimeMonitor responsible for the JobManager endpoint
- */
- CompletableFuture<Integer> requestWebPort(Time timeout);
-
- /**
* Submits a job to the JobManager.
*
* @param jobGraph to submit
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/StandaloneMiniCluster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/StandaloneMiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/StandaloneMiniCluster.java
index 07178fc..cbefe5a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/StandaloneMiniCluster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/StandaloneMiniCluster.java
@@ -92,6 +92,7 @@ public class StandaloneMiniCluster {
scheduledExecutorService,
scheduledExecutorService,
highAvailabilityServices,
+ Option.empty(),
JobManager.class,
MemoryArchivist.class);
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java
index 88c7f78..bd7f105 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java
@@ -56,18 +56,27 @@ import java.util.concurrent.TimeUnit;
public abstract class RestServerEndpoint {
protected final Logger log = LoggerFactory.getLogger(getClass());
+ private final Object lock = new Object();
+
private final String configuredAddress;
private final int configuredPort;
private final SSLEngine sslEngine;
private ServerBootstrap bootstrap;
private Channel serverChannel;
+ private String restAddress;
+
+ private volatile boolean started;
public RestServerEndpoint(RestServerEndpointConfiguration configuration) {
Preconditions.checkNotNull(configuration);
this.configuredAddress = configuration.getEndpointBindAddress();
this.configuredPort = configuration.getEndpointBindPort();
this.sslEngine = configuration.getSslEngine();
+
+ this.restAddress = null;
+
+ this.started = false;
}
/**
@@ -80,53 +89,65 @@ public abstract class RestServerEndpoint {
* Starts this REST server endpoint.
*/
public void start() {
- log.info("Starting rest endpoint.");
+ synchronized (lock) {
+ if (started) {
+ // RestServerEndpoint already started
+ return;
+ }
+
+ log.info("Starting rest endpoint.");
- final Router router = new Router();
+ final Router router = new Router();
- initializeHandlers().forEach(handler -> registerHandler(router, handler));
+ initializeHandlers().forEach(handler -> registerHandler(router, handler));
- ChannelInitializer<SocketChannel> initializer = new ChannelInitializer<SocketChannel>() {
+ ChannelInitializer<SocketChannel> initializer = new ChannelInitializer<SocketChannel>() {
- @Override
- protected void initChannel(SocketChannel ch) {
- Handler handler = new RouterHandler(router);
+ @Override
+ protected void initChannel(SocketChannel ch) {
+ Handler handler = new RouterHandler(router);
- // SSL should be the first handler in the pipeline
- if (sslEngine != null) {
- ch.pipeline().addLast("ssl", new SslHandler(sslEngine));
+ // SSL should be the first handler in the pipeline
+ if (sslEngine != null) {
+ ch.pipeline().addLast("ssl", new SslHandler(sslEngine));
+ }
+
+ ch.pipeline()
+ .addLast(new HttpServerCodec())
+ .addLast(new HttpObjectAggregator(1024 * 1024 * 10))
+ .addLast(handler.name(), handler)
+ .addLast(new PipelineErrorHandler(log));
}
+ };
+
+ NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("flink-rest-server-netty-boss"));
+ NioEventLoopGroup workerGroup = new NioEventLoopGroup(0, new DefaultThreadFactory("flink-rest-server-netty-worker"));
- ch.pipeline()
- .addLast(new HttpServerCodec())
- .addLast(new HttpObjectAggregator(1024 * 1024 * 10))
- .addLast(handler.name(), handler)
- .addLast(new PipelineErrorHandler(log));
+ bootstrap = new ServerBootstrap();
+ bootstrap
+ .group(bossGroup, workerGroup)
+ .channel(NioServerSocketChannel.class)
+ .childHandler(initializer);
+
+ final ChannelFuture channel;
+ if (configuredAddress == null) {
+ channel = bootstrap.bind(configuredPort);
+ } else {
+ channel = bootstrap.bind(configuredAddress, configuredPort);
}
- };
-
- NioEventLoopGroup bossGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("flink-rest-server-netty-boss"));
- NioEventLoopGroup workerGroup = new NioEventLoopGroup(0, new DefaultThreadFactory("flink-rest-server-netty-worker"));
-
- bootstrap = new ServerBootstrap();
- bootstrap
- .group(bossGroup, workerGroup)
- .channel(NioServerSocketChannel.class)
- .childHandler(initializer);
-
- final ChannelFuture channel;
- if (configuredAddress == null) {
- channel = bootstrap.bind(configuredPort);
- } else {
- channel = bootstrap.bind(configuredAddress, configuredPort);
- }
- serverChannel = channel.syncUninterruptibly().channel();
+ serverChannel = channel.syncUninterruptibly().channel();
+
+ InetSocketAddress bindAddress = (InetSocketAddress) serverChannel.localAddress();
+ String address = bindAddress.getAddress().getHostAddress();
+ int port = bindAddress.getPort();
- InetSocketAddress bindAddress = (InetSocketAddress) serverChannel.localAddress();
- String address = bindAddress.getAddress().getHostAddress();
- int port = bindAddress.getPort();
+ log.info("Rest endpoint listening at {}" + ':' + "{}", address, port);
- log.info("Rest endpoint listening at {}" + ':' + "{}", address, port);
+ // TODO: Make it include the protocol (http/https)
+ restAddress = address + ':' + port;
+
+ started = true;
+ }
}
/**
@@ -135,7 +156,9 @@ public abstract class RestServerEndpoint {
* @return address on which this endpoint is accepting requests
*/
public InetSocketAddress getServerAddress() {
+ Preconditions.checkState(started, "The RestServerEndpoint has not been started yet.");
Channel server = this.serverChannel;
+
if (server != null) {
try {
return ((InetSocketAddress) server.localAddress());
@@ -148,40 +171,65 @@ public abstract class RestServerEndpoint {
}
/**
+ * Returns the address of the REST server endpoint. Since the address is only known
+ * after the endpoint is started, it is returned as a future which is completed
+ * with the REST address at start up.
+ *
+ * @return REST address of this endpoint
+ */
+ public String getRestAddress() {
+ Preconditions.checkState(started, "The RestServerEndpoint has not been started yet.");
+ return restAddress;
+ }
+
+ /**
* Stops this REST server endpoint.
*/
public void shutdown(Time timeout) {
- log.info("Shutting down rest endpoint.");
- CompletableFuture<?> channelFuture = new CompletableFuture<>();
- if (this.serverChannel != null) {
- this.serverChannel.close().addListener(ignored -> channelFuture.complete(null));
- }
- CompletableFuture<?> groupFuture = new CompletableFuture<>();
- CompletableFuture<?> childGroupFuture = new CompletableFuture<>();
-
- channelFuture.thenRun(() -> {
- if (bootstrap != null) {
- if (bootstrap.group() != null) {
- bootstrap.group().shutdownGracefully(0, timeout.toMilliseconds(), TimeUnit.MILLISECONDS)
- .addListener(ignored -> groupFuture.complete(null));
- }
- if (bootstrap.childGroup() != null) {
- bootstrap.childGroup().shutdownGracefully(0, timeout.toMilliseconds(), TimeUnit.MILLISECONDS)
- .addListener(ignored -> childGroupFuture.complete(null));
+ synchronized (lock) {
+ if (!started) {
+ // RestServerEndpoint has not been started
+ return;
+ }
+
+ log.info("Shutting down rest endpoint.");
+
+ CompletableFuture<?> channelFuture = new CompletableFuture<>();
+ if (this.serverChannel != null) {
+ this.serverChannel.close().addListener(ignored -> channelFuture.complete(null));
+ serverChannel = null;
+ }
+ CompletableFuture<?> groupFuture = new CompletableFuture<>();
+ CompletableFuture<?> childGroupFuture = new CompletableFuture<>();
+
+ channelFuture.thenRun(() -> {
+ if (bootstrap != null) {
+ if (bootstrap.group() != null) {
+ bootstrap.group().shutdownGracefully(0, timeout.toMilliseconds(), TimeUnit.MILLISECONDS)
+ .addListener(ignored -> groupFuture.complete(null));
+ }
+ if (bootstrap.childGroup() != null) {
+ bootstrap.childGroup().shutdownGracefully(0, timeout.toMilliseconds(), TimeUnit.MILLISECONDS)
+ .addListener(ignored -> childGroupFuture.complete(null));
+ }
+ bootstrap = null;
+ } else {
+ // complete the group futures since there is nothing to stop
+ groupFuture.complete(null);
+ childGroupFuture.complete(null);
}
- } else {
- // complete the group futures since there is nothing to stop
- groupFuture.complete(null);
- childGroupFuture.complete(null);
+ });
+
+ try {
+ CompletableFuture.allOf(groupFuture, childGroupFuture).get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);
+ log.info("Rest endpoint shutdown complete.");
+ } catch (Exception e) {
+ log.warn("Rest endpoint shutdown failed.", e);
}
- });
- try {
- CompletableFuture.allOf(groupFuture, childGroupFuture).get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);
- log.info("Rest endpoint shutdown complete.");
- } catch (Exception e) {
- log.warn("Rest endpoint shutdown failed.", e);
+ restAddress = null;
+ started = false;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RedirectHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RedirectHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RedirectHandler.java
index 9f63456..fb40035 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RedirectHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RedirectHandler.java
@@ -40,6 +40,7 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
+import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
@@ -102,37 +103,30 @@ public abstract class RedirectHandler<T extends RestfulGateway> extends SimpleCh
optLeaderConsumer.ifPresent(
(T gateway) -> {
- OptionalConsumer<CompletableFuture<String>> optRedirectAddressConsumer = OptionalConsumer.of(
- HandlerRedirectUtils.getRedirectAddress(
- localAddress,
- gateway,
- timeout));
-
- optRedirectAddressConsumer
- .ifPresent(
- (CompletableFuture<String> redirectAddressFuture) ->
- redirectAddressFuture.whenComplete(
- (String redirectAddress, Throwable throwable) -> {
- if (throwable != null) {
- logger.error("Could not retrieve the redirect address.", throwable);
-
- HandlerUtils.sendErrorResponse(
- channelHandlerContext,
- routed.request(),
- new ErrorResponseBody("Could not retrieve the redirect address of the current leader. Please try to refresh."),
- HttpResponseStatus.INTERNAL_SERVER_ERROR);
- } else {
- HttpResponse response = HandlerRedirectUtils.getRedirectResponse(
- redirectAddress,
- routed.path(),
- httpsEnabled);
-
- KeepAliveWrite.flush(channelHandlerContext, routed.request(), response);
- }
- }
- ))
- .ifNotPresent(
- () -> {
+ CompletableFuture<Optional<String>> optRedirectAddressFuture = HandlerRedirectUtils.getRedirectAddress(
+ localAddress,
+ gateway,
+ timeout);
+
+ optRedirectAddressFuture.whenComplete(
+ (Optional<String> optRedirectAddress, Throwable throwable) -> {
+ HttpResponse response;
+ if (throwable != null) {
+ logger.error("Could not retrieve the redirect address.", throwable);
+
+ HandlerUtils.sendErrorResponse(
+ channelHandlerContext,
+ routed.request(),
+ new ErrorResponseBody("Could not retrieve the redirect address of the current leader. Please try to refresh."),
+ HttpResponseStatus.INTERNAL_SERVER_ERROR);
+ } else if (optRedirectAddress.isPresent()) {
+ response = HandlerRedirectUtils.getRedirectResponse(
+ optRedirectAddress.get(),
+ routed.path(),
+ httpsEnabled);
+
+ KeepAliveWrite.flush(channelHandlerContext, routed.request(), response);
+ } else {
try {
respondAsLeader(channelHandlerContext, routed, gateway);
} catch (Exception e) {
@@ -143,7 +137,9 @@ public abstract class RedirectHandler<T extends RestfulGateway> extends SimpleCh
new ErrorResponseBody("Error while responding to the request."),
HttpResponseStatus.INTERNAL_SERVER_ERROR);
}
- });
+ }
+ }
+ );
}
).ifNotPresent(
() ->
@@ -156,8 +152,8 @@ public abstract class RedirectHandler<T extends RestfulGateway> extends SimpleCh
HandlerUtils.sendErrorResponse(
channelHandlerContext,
routed.request(),
- new ErrorResponseBody("Service temporarily unavailable due to an ongoing leader election. Please refresh."),
- HttpResponseStatus.SERVICE_UNAVAILABLE);
+ new ErrorResponseBody("Local address has not been resolved. This indicates an internal error."),
+ HttpResponseStatus.INTERNAL_SERVER_ERROR);
}
} catch (Throwable throwable) {
logger.warn("Error occurred while processing web request.", throwable);
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/util/HandlerRedirectUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/util/HandlerRedirectUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/util/HandlerRedirectUtils.java
index 6f988a4..cd32d46 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/util/HandlerRedirectUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/util/HandlerRedirectUtils.java
@@ -20,7 +20,6 @@ package org.apache.flink.runtime.rest.handler.util;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.webmonitor.RestfulGateway;
import org.apache.flink.util.ExceptionUtils;
@@ -36,6 +35,7 @@ import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpVersion;
import javax.annotation.Nullable;
import java.nio.charset.Charset;
+import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
@@ -52,23 +52,19 @@ public class HandlerRedirectUtils {
public static final Charset ENCODING = ConfigConstants.DEFAULT_CHARSET;
- public static Optional<CompletableFuture<String>> getRedirectAddress(
- String localJobManagerAddress,
+ public static CompletableFuture<Optional<String>> getRedirectAddress(
+ String localRestAddress,
RestfulGateway restfulGateway,
Time timeout) {
- final String leaderAddress = restfulGateway.getAddress();
-
- final String jobManagerName = localJobManagerAddress.substring(localJobManagerAddress.lastIndexOf("/") + 1);
-
- if (!localJobManagerAddress.equals(leaderAddress) &&
- !leaderAddress.equals(AkkaUtils.getLocalAkkaURL(jobManagerName))) {
-
- return Optional.of(restfulGateway.requestRestAddress(timeout));
-
- } else {
- return Optional.empty();
- }
+ return restfulGateway.requestRestAddress(timeout).thenApply(
+ (String remoteRestAddress) -> {
+ if (Objects.equals(localRestAddress, remoteRestAddress)) {
+ return Optional.empty();
+ } else {
+ return Optional.of(remoteRestAddress);
+ }
+ });
}
public static HttpResponse getRedirectResponse(String redirectAddress, String path, boolean httpsEnabled) {
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitor.java
index 47cdb84..1e3c60f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitor.java
@@ -28,7 +28,7 @@ public interface WebMonitor {
*
* @throws Exception This method may forward exceptions, if it cannot bring up the web monitor.
*/
- void start(String jobManagerAkkaUrl) throws Exception;
+ void start() throws Exception;
/**
* Stops the web server.
@@ -45,4 +45,11 @@ public interface WebMonitor {
* @return The port where the web server is listening, or -1, if no server is running.
*/
int getServerPort();
+
+ /**
+ * Returns the REST address of this WebMonitor.
+ *
+ * @return REST address of this WebMonitor
+ */
+ String getRestAddress();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/scala/org/apache/flink/runtime/clusterframework/ContaineredJobManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/clusterframework/ContaineredJobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/clusterframework/ContaineredJobManager.scala
index 61c61b4..74ef1de 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/clusterframework/ContaineredJobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/clusterframework/ContaineredJobManager.scala
@@ -74,7 +74,8 @@ abstract class ContaineredJobManager(
submittedJobGraphs : SubmittedJobGraphStore,
checkpointRecoveryFactory : CheckpointRecoveryFactory,
jobRecoveryTimeout: FiniteDuration,
- metricsRegistry: Option[FlinkMetricRegistry])
+ metricsRegistry: Option[FlinkMetricRegistry],
+ optRestAddress: Option[String])
extends JobManager(
flinkConfiguration,
futureExecutor,
@@ -90,7 +91,8 @@ abstract class ContaineredJobManager(
submittedJobGraphs,
checkpointRecoveryFactory,
jobRecoveryTimeout,
- metricsRegistry) {
+ metricsRegistry,
+ optRestAddress) {
val jobPollingInterval: FiniteDuration
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index a3e1f08..3989fe4 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -85,7 +85,7 @@ import org.apache.flink.runtime.util._
import org.apache.flink.runtime.webmonitor.retriever.impl.{AkkaJobManagerRetriever, AkkaQueryServiceRetriever}
import org.apache.flink.runtime.webmonitor.{WebMonitor, WebMonitorUtils}
import org.apache.flink.runtime.{FlinkActor, LeaderSessionMessageFilter, LogMessages}
-import org.apache.flink.util.{InstantiationUtil, NetUtils, SerializedThrowable}
+import org.apache.flink.util.{FlinkException, InstantiationUtil, NetUtils, SerializedThrowable}
import scala.collection.JavaConverters._
import scala.collection.mutable
@@ -135,7 +135,8 @@ class JobManager(
protected val submittedJobGraphs : SubmittedJobGraphStore,
protected val checkpointRecoveryFactory : CheckpointRecoveryFactory,
protected val jobRecoveryTimeout: FiniteDuration,
- protected val metricsRegistry: Option[FlinkMetricRegistry])
+ protected val metricsRegistry: Option[FlinkMetricRegistry],
+ protected val optRestAddress: Option[String])
extends FlinkActor
with LeaderSessionMessageFilter // mixin oder is important, we want filtering after logging
with LogMessages // mixin order is important, we want first logging
@@ -164,16 +165,6 @@ class JobManager(
/** Futures which have to be completed before terminating the job manager */
var futuresToComplete: Option[Seq[Future[Unit]]] = None
- /**
- * The port of the web monitor as configured. Make sure that it is actually configured before
- * starting the JobManager. This tightly couples the web monitor with the job manager. It is a
- * temporary workaround until all execution graph components are properly serializable and all
- * web monitors can transparently interact with each job manager. Currently each web server has
- * to run in the actor system of the associated job manager.
- */
- val webMonitorPort : Int = flinkConfiguration.getInteger(
- WebOptions.PORT, -1)
-
/** The default directory for savepoints. */
val defaultSavepointDir: String = flinkConfiguration.getString(CoreOptions.SAVEPOINT_DIRECTORY)
@@ -1208,8 +1199,13 @@ class JobManager(
case RequestLeaderSessionID =>
sender() ! ResponseLeaderSessionID(leaderSessionID.orNull)
- case RequestWebMonitorPort =>
- sender() ! ResponseWebMonitorPort(webMonitorPort)
+ case RequestRestAddress =>
+ sender() !
+ decorateMessage(
+ optRestAddress.getOrElse(
+ Status.Failure(
+ new FlinkException("No REST endpoint has been started for the JobManager.")))
+ )
}
/**
@@ -2253,6 +2249,7 @@ object JobManager {
futureExecutor,
ioExecutor,
highAvailabilityServices,
+ webMonitor.map(_.getRestAddress),
jobManagerClass,
archiveClass)
@@ -2293,15 +2290,7 @@ object JobManager {
// start web monitor
webMonitor.foreach {
- monitor =>
- val hostnamePort = HighAvailabilityServicesUtils.getJobManagerAddress(configuration)
- val jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
- hostnamePort.f0,
- hostnamePort.f1,
- JobMaster.JOB_MANAGER_NAME,
- AddressResolution.NO_ADDRESS_RESOLUTION,
- configuration)
- monitor.start(jobManagerAkkaUrl)
+ _.start()
}
val resourceManager =
@@ -2571,6 +2560,8 @@ object JobManager {
* @param actorSystem The actor system running the JobManager
* @param futureExecutor to run JobManager's futures
* @param ioExecutor to run blocking io operations
+ * @param highAvailabilityServices high availability services
+ * @param optRestAddress optional rest address
* @param jobManagerClass The class of the JobManager to be started
* @param archiveClass The class of the MemoryArchivist to be started
* @return A tuple of references (JobManager Ref, Archiver Ref)
@@ -2581,6 +2572,7 @@ object JobManager {
futureExecutor: ScheduledExecutorService,
ioExecutor: Executor,
highAvailabilityServices: HighAvailabilityServices,
+ optRestAddress: Option[String],
jobManagerClass: Class[_ <: JobManager],
archiveClass: Class[_ <: MemoryArchivist])
: (ActorRef, ActorRef) = {
@@ -2591,6 +2583,7 @@ object JobManager {
futureExecutor,
ioExecutor,
highAvailabilityServices,
+ optRestAddress,
Some(JobMaster.JOB_MANAGER_NAME),
Some(JobMaster.ARCHIVE_NAME),
jobManagerClass,
@@ -2605,6 +2598,8 @@ object JobManager {
* @param actorSystem The actor system running the JobManager
* @param futureExecutor to run JobManager's futures
* @param ioExecutor to run blocking io operations
+ * @param highAvailabilityServices high availability services
+ * @param optRestAddress optional rest address
* @param jobManagerActorName Optionally the name of the JobManager actor. If none is given,
* the actor will have the name generated by the actor system.
* @param archiveActorName Optionally the name of the archive actor. If none is given,
@@ -2619,6 +2614,7 @@ object JobManager {
futureExecutor: ScheduledExecutorService,
ioExecutor: Executor,
highAvailabilityServices: HighAvailabilityServices,
+ optRestAddress: Option[String],
jobManagerActorName: Option[String],
archiveActorName: Option[String],
jobManagerClass: Class[_ <: JobManager],
@@ -2665,7 +2661,8 @@ object JobManager {
highAvailabilityServices.getSubmittedJobGraphStore(),
highAvailabilityServices.getCheckpointRecoveryFactory(),
jobRecoveryTimeout,
- metricsRegistry)
+ metricsRegistry,
+ optRestAddress)
val jobManager: ActorRef = jobManagerActorName match {
case Some(actorName) => actorSystem.actorOf(jobManagerProps, actorName)
@@ -2704,7 +2701,8 @@ object JobManager {
submittedJobGraphStore: SubmittedJobGraphStore,
checkpointRecoveryFactory: CheckpointRecoveryFactory,
jobRecoveryTimeout: FiniteDuration,
- metricsRegistry: Option[FlinkMetricRegistry]): Props = {
+ metricsRegistry: Option[FlinkMetricRegistry],
+ optRestAddress: Option[String]): Props = {
Props(
jobManagerClass,
@@ -2722,6 +2720,7 @@ object JobManager {
submittedJobGraphStore,
checkpointRecoveryFactory,
jobRecoveryTimeout,
- metricsRegistry)
+ metricsRegistry,
+ optRestAddress)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/JobManagerMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/JobManagerMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/JobManagerMessages.scala
index 52f3777..7d13d9f 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/JobManagerMessages.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/JobManagerMessages.scala
@@ -464,15 +464,8 @@ object JobManagerMessages {
/** Response containing the ActorRef of the archiver */
case class ResponseArchive(actor: ActorRef)
- /** Request for the [[org.apache.flink.runtime.webmonitor.WebMonitor]] port. */
- case object RequestWebMonitorPort
-
- /**
- * Response containing the [[org.apache.flink.runtime.webmonitor.WebMonitor]] port.
- *
- * -1 indicates that there is no web monitor running.
- */
- case class ResponseWebMonitorPort(port: Integer)
+ /** Request for the JobManager's REST endpoint address */
+ case object RequestRestAddress
/**
* Triggers a savepoint for the specified job.
@@ -580,8 +573,8 @@ object JobManagerMessages {
RecoverAllJobs
}
- def getRequestWebMonitorPort: AnyRef = {
- RequestWebMonitorPort
+ def getRequestRestAddress: AnyRef = {
+ RequestRestAddress
}
def getDisposeSavepointSuccess: AnyRef = {
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
index c6cbb84..7f4308e 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
@@ -152,7 +152,11 @@ abstract class FlinkMiniCluster(
def startResourceManager(index: Int, system: ActorSystem): ActorRef
- def startJobManager(index: Int, system: ActorSystem): ActorRef
+ def startJobManager(
+ index: Int,
+ system: ActorSystem,
+ optRestAddress: Option[String])
+ : ActorRef
def startTaskManager(index: Int, system: ActorSystem): ActorRef
@@ -328,7 +332,12 @@ abstract class FlinkMiniCluster(
} else {
startJobManagerActorSystem(i)
}
- (actorSystem, startJobManager(i, actorSystem))
+
+ if (i == 0) {
+ webMonitor = startWebServer(originalConfiguration, actorSystem)
+ }
+
+ (actorSystem, startJobManager(i, actorSystem, webMonitor.map(_.getRestAddress)))
}).unzip
jobManagerActorSystems = Some(jmActorSystems)
@@ -370,10 +379,6 @@ abstract class FlinkMiniCluster(
taskManagerActorSystems = Some(tmActorSystems)
taskManagerActors = Some(tmActors)
- val jobManagerAkkaURL = AkkaUtils.getAkkaURL(jmActorSystems(0), jmActors(0))
-
- webMonitor = startWebServer(originalConfiguration, jmActorSystems(0), jobManagerAkkaURL)
-
if(waitForTaskManagerRegistration) {
waitForTaskManagersToBeRegistered()
}
@@ -383,8 +388,7 @@ abstract class FlinkMiniCluster(
def startWebServer(
config: Configuration,
- actorSystem: ActorSystem,
- jobManagerAkkaURL: String)
+ actorSystem: ActorSystem)
: Option[WebMonitor] = {
if(
config.getBoolean(ConfigConstants.LOCAL_START_WEBSERVER, false) &&
@@ -405,7 +409,7 @@ abstract class FlinkMiniCluster(
actorSystem.dispatcher)
)
- webServer.foreach(_.start(jobManagerAkkaURL))
+ webServer.foreach(_.start())
webServer
} else {
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
index dcf9dd0..e22230e 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
@@ -120,7 +120,11 @@ class LocalFlinkMiniCluster(
// Start methods for the distributed components
//------------------------------------------------------------------------------------------------
- override def startJobManager(index: Int, system: ActorSystem): ActorRef = {
+ override def startJobManager(
+ index: Int,
+ system: ActorSystem,
+ optRestAddress: Option[String])
+ : ActorRef = {
val config = originalConfiguration.clone()
val jobManagerName = getJobManagerName(index)
@@ -176,7 +180,8 @@ class LocalFlinkMiniCluster(
highAvailabilityServices.getSubmittedJobGraphStore(),
highAvailabilityServices.getCheckpointRecoveryFactory(),
jobRecoveryTimeout,
- metricsRegistry),
+ metricsRegistry,
+ optRestAddress),
jobManagerName)
}
@@ -291,7 +296,8 @@ class LocalFlinkMiniCluster(
submittedJobGraphStore: SubmittedJobGraphStore,
checkpointRecoveryFactory: CheckpointRecoveryFactory,
jobRecoveryTimeout: FiniteDuration,
- metricsRegistry: Option[MetricRegistry])
+ metricsRegistry: Option[MetricRegistry],
+ optRestAddress: Option[String])
: Props = {
JobManager.getJobManagerProps(
@@ -310,7 +316,8 @@ class LocalFlinkMiniCluster(
submittedJobGraphStore,
checkpointRecoveryFactory,
jobRecoveryTimeout,
- metricsRegistry)
+ metricsRegistry,
+ optRestAddress)
}
def getTaskManagerProps(
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java
index da76115..0b6450d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java
@@ -47,6 +47,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
+import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
@@ -204,7 +205,8 @@ public class DispatcherTest extends TestLogger {
blobServer,
heartbeatServices,
metricRegistry,
- fatalErrorHandler);
+ fatalErrorHandler,
+ Optional.empty());
this.jobManagerRunner = jobManagerRunner;
this.expectedJobId = expectedJobId;
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
index 173730a..72f3a88 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
@@ -203,7 +203,8 @@ public class JobManagerHARecoveryTest extends TestLogger {
mySubmittedJobGraphStore,
checkpointStateFactory,
jobRecoveryTimeout,
- Option.apply(null));
+ Option.<MetricRegistry>empty(),
+ Option.<String>empty());
jobManager = system.actorOf(jobManagerProps);
ActorGateway gateway = new AkkaActorGateway(jobManager, leaderSessionID);
@@ -430,7 +431,8 @@ public class JobManagerHARecoveryTest extends TestLogger {
submittedJobGraphs,
checkpointRecoveryFactory,
jobRecoveryTimeout,
- metricsRegistry);
+ metricsRegistry,
+ Option.empty());
this.recoveredJobs = recoveredJobs;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
index e863484..8b88e11 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
@@ -611,6 +611,7 @@ public class JobManagerTest extends TestLogger {
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
highAvailabilityServices,
+ Option.empty(),
TestingJobManager.class,
MemoryArchivist.class)._1();
@@ -827,6 +828,7 @@ public class JobManagerTest extends TestLogger {
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
highAvailabilityServices,
+ Option.empty(),
Option.apply("jm"),
Option.apply("arch"),
TestingJobManager.class,
@@ -962,6 +964,7 @@ public class JobManagerTest extends TestLogger {
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
highAvailabilityServices,
+ Option.empty(),
Option.apply("jm"),
Option.apply("arch"),
TestingJobManager.class,
@@ -1073,6 +1076,7 @@ public class JobManagerTest extends TestLogger {
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
highAvailabilityServices,
+ Option.empty(),
Option.apply("jm"),
Option.apply("arch"),
TestingJobManager.class,
@@ -1182,6 +1186,7 @@ public class JobManagerTest extends TestLogger {
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
highAvailabilityServices,
+ Option.empty(),
Option.apply("jm"),
Option.apply("arch"),
TestingJobManager.class,
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
index 6a39293..c79d3ce 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
@@ -45,6 +45,8 @@ import org.apache.flink.util.NetUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+
+import scala.Option;
import scala.Tuple2;
import scala.concurrent.Await;
import scala.concurrent.Future;
@@ -94,6 +96,7 @@ public class JobSubmitTest {
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
highAvailabilityServices,
+ Option.empty(),
JobManager.class,
MemoryArchivist.class)._1();
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/JobManagerLeaderElectionTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/JobManagerLeaderElectionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/JobManagerLeaderElectionTest.java
index 230ca91..b853b14 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/JobManagerLeaderElectionTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/JobManagerLeaderElectionTest.java
@@ -42,6 +42,7 @@ import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
import org.apache.flink.runtime.jobmanager.StandaloneSubmittedJobGraphStore;
import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
+import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.testingUtils.TestingJobManager;
import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages;
import org.apache.flink.runtime.testingUtils.TestingUtils;
@@ -196,7 +197,7 @@ public class JobManagerLeaderElectionTest extends TestLogger {
submittedJobGraphStore,
checkpointRecoveryFactory,
AkkaUtils.getDefaultTimeoutAsFiniteDuration(),
- Option.apply(null)
- );
+ Option.<MetricRegistry>empty(),
+ Option.<String>empty());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/TaskManagerMetricsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/TaskManagerMetricsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/TaskManagerMetricsTest.java
index 5e6bbce..cea0928 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/TaskManagerMetricsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/TaskManagerMetricsTest.java
@@ -44,6 +44,7 @@ import java.net.InetAddress;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import scala.Option;
import scala.concurrent.duration.FiniteDuration;
/**
@@ -72,6 +73,7 @@ public class TaskManagerMetricsTest extends TestLogger {
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
highAvailabilityServices,
+ Option.empty(),
JobManager.class,
MemoryArchivist.class)._1();
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestEndpointITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestEndpointITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestEndpointITCase.java
index 89d87f8..ff1c1f7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestEndpointITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestEndpointITCase.java
@@ -51,6 +51,7 @@ import java.util.Collections;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -73,7 +74,7 @@ public class RestEndpointITCase extends TestLogger {
final String address = "localhost";
RestfulGateway mockRestfulGateway = mock(RestfulGateway.class);
- when(mockRestfulGateway.getAddress()).thenReturn(address);
+ when(mockRestfulGateway.requestRestAddress(any(Time.class))).thenReturn(CompletableFuture.completedFuture(address));
GatewayRetriever<RestfulGateway> mockGatewayRetriever = mock(GatewayRetriever.class);
when(mockGatewayRetriever.getNow()).thenReturn(Optional.of(mockRestfulGateway));
http://git-wip-us.apache.org/repos/asf/flink/blob/6a62f145/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
index 7234fea..410b93e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
@@ -46,7 +46,6 @@ import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
import org.apache.flink.runtime.io.network.partition.ResultPartitionManager;
import org.apache.flink.runtime.jobmanager.JobManager;
import org.apache.flink.runtime.jobmanager.MemoryArchivist;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.memory.MemoryManager;
import org.apache.flink.runtime.messages.TaskManagerMessages;
import org.apache.flink.runtime.metrics.MetricRegistry;
@@ -58,6 +57,7 @@ import org.apache.flink.runtime.testingUtils.TestingUtils;
import org.apache.flink.util.TestLogger;
import org.junit.Test;
+import scala.Option;
import scala.concurrent.duration.FiniteDuration;
import java.net.InetAddress;
@@ -98,6 +98,7 @@ public class TaskManagerComponentsStartupShutdownTest extends TestLogger {
TestingUtils.defaultExecutor(),
TestingUtils.defaultExecutor(),
highAvailabilityServices,
+ Option.empty(),
JobManager.class,
MemoryArchivist.class)._1();