You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2016/11/01 08:40:28 UTC
[01/50] [abbrv] flink git commit: [FLINK-4537] rebase and refine
[Forced Update!]
Repository: flink
Updated Branches:
refs/heads/flip-6 80b6c2a01 -> 4c4d8eb14 (forced update)
[FLINK-4537] rebase and refine
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d16025d8
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d16025d8
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d16025d8
Branch: refs/heads/flip-6
Commit: d16025d8b1d95eb1df1f32e11b3b022d3c9cc541
Parents: 84c84b4
Author: Maximilian Michels <mx...@apache.org>
Authored: Wed Sep 21 14:13:12 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:26 2016 +0100
----------------------------------------------------------------------
.../resourcemanager/JobMasterRegistration.java | 52 +++---
.../resourcemanager/ResourceManager.java | 165 ++++++++-----------
.../slotmanager/SlotManager.java | 29 +++-
3 files changed, 110 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/d16025d8/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobMasterRegistration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobMasterRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobMasterRegistration.java
index 7b8ec70..981441f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobMasterRegistration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobMasterRegistration.java
@@ -18,59 +18,47 @@
package org.apache.flink.runtime.resourcemanager;
-<<<<<<< HEAD
import org.apache.flink.api.common.JobID;
-=======
import org.apache.flink.runtime.jobmaster.JobMasterGateway;
->>>>>>> db98efb... rsourceManager registration with JobManager
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
-import java.io.Serializable;
import java.util.UUID;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
/**
* This class is responsible for group the JobMasterGateway and the LeaderSessionID of a registered job master
*/
-public class JobMasterRegistration implements Serializable {
+public class JobMasterRegistration implements LeaderRetrievalListener {
-<<<<<<< HEAD
- private final String address;
+ private final JobMasterGateway gateway;
private final JobID jobID;
+ private final UUID leaderSessionID;
+ private LeaderRetrievalListener retriever;
- public JobMasterRegistration(String address, JobID jobID) {
- this.address = address;
+ public JobMasterRegistration(JobMasterGateway gateway, JobID jobID, UUID leaderSessionID) {
+ this.gateway = gateway;
this.jobID = jobID;
-=======
- private static final long serialVersionUID = -2316627821716999527L;
-
- private final JobMasterGateway jobMasterGateway;
-
- private UUID jobMasterLeaderSessionID;
-
- public JobMasterRegistration(JobMasterGateway jobMasterGateway) {
- this.jobMasterGateway = checkNotNull(jobMasterGateway);
+ this.leaderSessionID = leaderSessionID;
}
- public JobMasterRegistration(JobMasterGateway jobMasterGateway, UUID jobMasterLeaderSessionID) {
- this.jobMasterGateway = checkNotNull(jobMasterGateway);
- this.jobMasterLeaderSessionID = jobMasterLeaderSessionID;
+ public JobMasterGateway getGateway() {
+ return gateway;
}
- public JobMasterGateway getJobMasterGateway() {
- return jobMasterGateway;
+ public UUID getLeaderSessionID() {
+ return leaderSessionID;
}
- public void setJobMasterLeaderSessionID(UUID leaderSessionID) {
- this.jobMasterLeaderSessionID = jobMasterLeaderSessionID;
->>>>>>> db98efb... rsourceManager registration with JobManager
+ public JobID getJobID() {
+ return jobID;
}
- public UUID getJobMasterLeaderSessionID() {
- return jobMasterLeaderSessionID;
+ @Override
+ public void notifyLeaderAddress(String leaderAddress, UUID leaderSessionID) {
+
}
- public JobID getJobID() {
- return jobID;
+ @Override
+ public void handleError(Exception exception) {
+
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d16025d8/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index 8be1455..aae4874 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.resourcemanager;
-import akka.dispatch.Futures;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
@@ -29,26 +28,31 @@ import org.apache.flink.runtime.instance.InstanceID;
import org.apache.flink.runtime.jobmaster.JobMasterRegistrationSuccess;
import org.apache.flink.runtime.leaderelection.LeaderContender;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
-<<<<<<< HEAD
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
-=======
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
->>>>>>> db98efb... rsourceManager registration with JobManager
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.jobmaster.JobMaster;
import org.apache.flink.runtime.jobmaster.JobMasterGateway;
-import org.apache.flink.runtime.rpc.exceptions.LeaderSessionIDException;
import org.apache.flink.runtime.taskexecutor.TaskExecutorRegistrationSuccess;
import org.apache.flink.runtime.registration.RegistrationResponse;
+import org.apache.flink.runtime.concurrent.Future;
+
+import org.apache.flink.runtime.util.LeaderConnectionInfo;
+import org.apache.flink.runtime.util.LeaderRetrievalUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import scala.concurrent.duration.FiniteDuration;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -62,17 +66,13 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
* <li>{@link #requestSlot(SlotRequest)} requests a slot from the resource manager</li>
* </ul>
*/
-<<<<<<< HEAD
public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> implements LeaderContender {
private final Logger LOG = LoggerFactory.getLogger(getClass());
private final Map<JobID, JobMasterGateway> jobMasterGateways;
-=======
-public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> {
- /** the mapping relationship of JobID and JobMasterGateway */
- private final Map<JobID, JobMasterRegistration> jobMasters;
->>>>>>> db98efb... rsourceManager registration with JobManager
+
+ private final Set<LeaderRetrievalListener> jobMasterLeaderRetrievalListeners;
private final HighAvailabilityServices highAvailabilityServices;
@@ -88,12 +88,9 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> {
SlotManager slotManager) {
super(rpcService);
this.highAvailabilityServices = checkNotNull(highAvailabilityServices);
-<<<<<<< HEAD
this.jobMasterGateways = new HashMap<>();
this.slotManager = slotManager;
-=======
- this.jobMasters = new HashMap<>(16);
->>>>>>> db98efb... rsourceManager registration with JobManager
+ this.jobMasterLeaderRetrievalListeners = new HashSet<>();
}
@Override
@@ -113,7 +110,7 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> {
public void shutDown() {
try {
leaderElectionService.stop();
- for(JobID jobID : jobMasters.keySet()) {
+ for(JobID jobID : jobMasterGateways.keySet()) {
highAvailabilityServices.getJobMasterLeaderRetriever(jobID).stop();
}
super.shutDown();
@@ -142,52 +139,64 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> {
* @return Future registration response
*/
@RpcMethod
-<<<<<<< HEAD
- public Future<RegistrationResponse> registerJobMaster(JobMasterRegistration jobMasterRegistration) {
- final Future<JobMasterGateway> jobMasterFuture =
- getRpcService().connect(jobMasterRegistration.getAddress(), JobMasterGateway.class);
- final JobID jobID = jobMasterRegistration.getJobID();
-=======
- public Future<RegistrationResponse> registerJobMaster(UUID resourceManagerLeaderId, final String jobMasterAddress, final JobID jobID) {
+ public Future<RegistrationResponse> registerJobMaster(
+ final UUID resourceManagerLeaderId, final UUID jobMasterLeaderId,
+ final String jobMasterAddress, final JobID jobID) {
+
+ checkNotNull(resourceManagerLeaderId);
+ checkNotNull(jobMasterAddress);
+ checkNotNull(jobID);
+
+ // TODO mxm The leader retrieval needs to be split up in an async part which runs outside the main execution thread
+ // The state updates should be performed inside the main thread
+
+ final FlinkCompletableFuture<RegistrationResponse> future = new FlinkCompletableFuture<>();
if(!leaderSessionID.equals(resourceManagerLeaderId)) {
- log.warn("Discard registration from JobMaster {} at ({}) because the expected leader session ID {} did not equal the received leader session ID {}",
+ log.warn("Discard registration from JobMaster {} at ({}) because the expected leader session ID {}" +
+ " did not equal the received leader session ID {}",
jobID, jobMasterAddress, leaderSessionID, resourceManagerLeaderId);
- return Futures.failed(new LeaderSessionIDException(leaderSessionID, resourceManagerLeaderId));
+ future.complete(new RegistrationResponse.Decline("Invalid leader session id"));
+ return future;
}
- Future<JobMasterGateway> jobMasterFuture = getRpcService().connect(jobMasterAddress, JobMasterGateway.class);
->>>>>>> db98efb... rsourceManager registration with JobManager
+ final LeaderConnectionInfo jobMasterLeaderInfo;
+ try {
+ jobMasterLeaderInfo = LeaderRetrievalUtils.retrieveLeaderConnectionInfo(
+ highAvailabilityServices.getJobMasterLeaderRetriever(jobID), new FiniteDuration(5, TimeUnit.SECONDS));
+ } catch (Exception e) {
+ LOG.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
+ future.complete(new RegistrationResponse.Decline("Failed to retrieve JobMasterLeaderRetriever"));
+ return future;
+ }
+
+ if (!jobMasterLeaderId.equals(jobMasterLeaderInfo.getLeaderSessionID())) {
+ LOG.info("Declining registration request from non-leading JobManager {}", jobMasterAddress);
+ future.complete(new RegistrationResponse.Decline("JobManager is not leading"));
+ return future;
+ }
- return jobMasterFuture.thenApplyAsync(new ApplyFunction<JobMasterGateway, RegistrationResponse>() {
+ Future<JobMasterGateway> jobMasterGatewayFuture =
+ getRpcService().connect(jobMasterAddress, JobMasterGateway.class);
+
+ return jobMasterGatewayFuture.thenApplyAsync(new ApplyFunction<JobMasterGateway, RegistrationResponse>() {
@Override
public RegistrationResponse apply(JobMasterGateway jobMasterGateway) {
-<<<<<<< HEAD
+
+ final JobMasterLeaderListener jobMasterLeaderListener = new JobMasterLeaderListener(jobID);
+ try {
+ LeaderRetrievalService jobMasterLeaderRetriever = highAvailabilityServices.getJobMasterLeaderRetriever(jobID);
+ jobMasterLeaderRetriever.start(jobMasterLeaderListener);
+ } catch (Exception e) {
+ LOG.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
+ return new RegistrationResponse.Decline("Failed to retrieve JobMasterLeaderRetriever");
+ }
+ jobMasterLeaderRetrievalListeners.add(jobMasterLeaderListener);
final JobMasterGateway existingGateway = jobMasterGateways.put(jobID, jobMasterGateway);
if (existingGateway != null) {
- LOG.info("Replacing existing gateway {} for JobID {} with {}.",
- existingGateway, jobID, jobMasterGateway);
- }
- return new RegistrationResponse(true);
-=======
- if (jobMasters.containsKey(jobID)) {
- JobMasterRegistration jobMasterRegistration = new JobMasterRegistration(jobMasterGateway, jobMasters.get(jobID).getJobMasterLeaderSessionID());
- jobMasters.put(jobID, jobMasterRegistration);
log.info("Replacing gateway for registered JobID {}.", jobID);
- } else {
- JobMasterRegistration jobMasterRegistration = new JobMasterRegistration(jobMasterGateway);
- jobMasters.put(jobID, jobMasterRegistration);
- try {
- highAvailabilityServices.getJobMasterLeaderRetriever(jobID).start(new JobMasterLeaderListener(jobID));
- } catch(Throwable e) {
- log.warn("Decline registration from JobMaster {} at ({}) because fail to get the leader retriever for the given job JobMaster",
- jobID, jobMasterAddress);
- return new RegistrationResponse.Decline("Fail to get the leader retriever for the given JobMaster");
- }
}
-
return new JobMasterRegistrationSuccess(5000);
->>>>>>> db98efb... rsourceManager registration with JobManager
}
}, getMainThreadExecutor());
}
@@ -228,26 +237,9 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> {
}
-<<<<<<< HEAD
// ------------------------------------------------------------------------
// Leader Contender
// ------------------------------------------------------------------------
-=======
- /**
- * Callback method when current resourceManager lose leadership.
- */
- @Override
- public void revokeLeadership() {
- runAsync(new Runnable() {
- @Override
- public void run() {
- log.info("ResourceManager {} was revoked leadership.", getAddress());
- jobMasters.clear();
- leaderSessionID = null;
- }
- });
- }
->>>>>>> db98efb... rsourceManager registration with JobManager
/**
* Callback method when current resourceManager is granted leadership
@@ -263,7 +255,7 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> {
// confirming the leader session ID might be blocking,
leaderElectionService.confirmLeaderSessionID(leaderSessionID);
// notify SlotManager
- slotManager.notifyLeaderAddress(getAddress(), leaderSessionID);
+ slotManager.setLeaderUUID(leaderSessionID);
ResourceManager.this.leaderSessionID = leaderSessionID;
}
});
@@ -279,7 +271,8 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> {
public void run() {
log.info("ResourceManager {} was revoked leadership.", getAddress());
jobMasterGateways.clear();
- ResourceManager.this.leaderSessionID = null;
+ slotManager.clearState();
+ leaderSessionID = null;
}
});
}
@@ -291,20 +284,15 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> {
*/
@Override
public void handleError(final Exception exception) {
- runAsync(new Runnable() {
- @Override
- public void run() {
- log.error("ResourceManager received an error from the LeaderElectionService.", exception);
- // notify SlotManager
- slotManager.handleError(exception);
- // terminate ResourceManager in case of an error
- shutDown();
- }
- });
+ log.error("ResourceManager received an error from the LeaderElectionService.", exception);
+ // terminate ResourceManager in case of an error
+ shutDown();
}
- private class JobMasterLeaderListener implements LeaderRetrievalListener {
+ private static class JobMasterLeaderListener implements LeaderRetrievalListener {
+
private final JobID jobID;
+ private UUID leaderID;
private JobMasterLeaderListener(JobID jobID) {
this.jobID = jobID;
@@ -312,25 +300,12 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> {
@Override
public void notifyLeaderAddress(final String leaderAddress, final UUID leaderSessionID) {
- runAsync(new Runnable() {
- @Override
- public void run() {
- log.info("A new leader for JobMaster {} is elected, address is {}, leaderSessionID is {}", jobID, leaderAddress, leaderSessionID);
- // update job master leader session id
- JobMasterRegistration jobMasterRegistration = jobMasters.get(jobID);
- jobMasterRegistration.setJobMasterLeaderSessionID(leaderSessionID);
- }
- });
+ this.leaderID = leaderSessionID;
}
@Override
public void handleError(final Exception exception) {
- runAsync(new Runnable() {
- @Override
- public void run() {
- log.error("JobMasterLeaderListener received an error from the LeaderRetrievalService", exception);
- }
- });
+ // TODO
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d16025d8/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
index 97176b2..5d0013c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
@@ -59,7 +59,7 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
* </ul>
* <b>IMPORTANT:</b> This class is <b>Not Thread-safe</b>.
*/
-public abstract class SlotManager implements LeaderRetrievalListener {
+public abstract class SlotManager {
protected final Logger LOG = LoggerFactory.getLogger(getClass());
@@ -514,22 +514,33 @@ public abstract class SlotManager implements LeaderRetrievalListener {
public int size() {
return allocatedSlots.size();
}
+
+ public void clear() {
+ allocatedSlots.clear();
+ allocatedSlotsByAllocationId.clear();
+ }
+ }
+
+ /**
+ * Clears the state of the SlotManager after leadership revokal
+ */
+ public void clearState() {
+ taskManagerGateways.clear();
+ registeredSlots.clear();
+ pendingSlotRequests.clear();
+ freeSlots.clear();
+ allocationMap.clear();
+ leaderID = null;
}
// ------------------------------------------------------------------------
- // High availability
+ // High availability (called by the ResourceManager)
// ------------------------------------------------------------------------
- @Override
- public void notifyLeaderAddress(String leaderAddress, UUID leaderSessionID) {
+ public void setLeaderUUID(UUID leaderSessionID) {
this.leaderID = leaderSessionID;
}
- @Override
- public void handleError(Exception exception) {
- LOG.error("Slot Manager received an error from the leader service", exception);
- }
-
// ------------------------------------------------------------------------
// Testing utilities
// ------------------------------------------------------------------------
[09/50] [abbrv] flink git commit: [FLINK-4703] RpcCompletenessTest:
Add support for type arguments and subclasses
Posted by tr...@apache.org.
[FLINK-4703] RpcCompletenessTest: Add support for type arguments and subclasses
This closes #2561
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ae3a8d5f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ae3a8d5f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ae3a8d5f
Branch: refs/heads/flip-6
Commit: ae3a8d5f883520d50372b04e5c5dd99ab6e4ad59
Parents: 359ee0a
Author: Maximilian Michels <mx...@apache.org>
Authored: Wed Sep 28 12:39:30 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:28 2016 +0100
----------------------------------------------------------------------
.../apache/flink/runtime/rpc/RpcEndpoint.java | 23 +++++-
.../flink/runtime/rpc/RpcCompletenessTest.java | 80 ++++++++++++++++++--
2 files changed, 94 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/ae3a8d5f/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
index 4e5e49a..79961f7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
@@ -85,9 +85,9 @@ public abstract class RpcEndpoint<C extends RpcGateway> {
// IMPORTANT: Don't change order of selfGatewayType and self because rpcService.startServer
// requires that selfGatewayType has been initialized
- this.selfGatewayType = ReflectionUtil.getTemplateType1(getClass());
+ this.selfGatewayType = determineSelfGatewayType();
this.self = rpcService.startServer(this);
-
+
this.mainThreadExecutor = new MainThreadExecutor((MainThreadExecutable) self);
}
@@ -255,4 +255,23 @@ public abstract class RpcEndpoint<C extends RpcGateway> {
gateway.runAsync(runnable);
}
}
+
+ /**
+ * Determines the self gateway type specified in one of the subclasses which extend this class.
+ * May traverse multiple class hierarchies until a Gateway type is found as a first type argument.
+ * @return Class<C> The determined self gateway type
+ */
+ private Class<C> determineSelfGatewayType() {
+
+ // determine self gateway type
+ Class c = getClass();
+ Class<C> determinedSelfGatewayType;
+ do {
+ determinedSelfGatewayType = ReflectionUtil.getTemplateType1(c);
+ // check if super class contains self gateway type in next loop
+ c = c.getSuperclass();
+ } while (!RpcGateway.class.isAssignableFrom(determinedSelfGatewayType));
+
+ return determinedSelfGatewayType;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ae3a8d5f/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcCompletenessTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcCompletenessTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcCompletenessTest.java
index 53355e8..e7143ae 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcCompletenessTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcCompletenessTest.java
@@ -26,9 +26,14 @@ import org.apache.flink.util.ReflectionUtil;
import org.apache.flink.util.TestLogger;
import org.junit.Test;
import org.reflections.Reflections;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.lang.annotation.Annotation;
import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.lang.reflect.Type;
+import java.lang.reflect.TypeVariable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -41,8 +46,33 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+/**
+ * Test which ensures that all classes of subtype {@link RpcEndpoint} implement
+ * the methods specified in the generic gateway type argument.
+ *
+ * {@code
+ * RpcEndpoint<GatewayTypeParameter extends RpcGateway>
+ * }
+ *
+ * Note, that the class hierarchy can also be nested. In this case the type argument
+ * always has to be the first argument, e.g. {@code
+ *
+ * // RpcClass needs to implement RpcGatewayClass' methods
+ * RpcClass extends RpcEndpoint<RpcGatewayClass>
+ *
+ * // RpcClass2 or its subclass needs to implement RpcGatewayClass' methods
+ * RpcClass<GatewayTypeParameter extends RpcGateway,...> extends RpcEndpoint<GatewayTypeParameter>
+ * RpcClass2 extends RpcClass<RpcGatewayClass,...>
+ *
+ * // needless to say, this can even be nested further
+ * ...
+ * }
+ *
+ */
public class RpcCompletenessTest extends TestLogger {
+ private static Logger LOG = LoggerFactory.getLogger(RpcCompletenessTest.class);
+
private static final Class<?> futureClass = Future.class;
private static final Class<?> timeoutClass = Time.class;
@@ -55,16 +85,52 @@ public class RpcCompletenessTest extends TestLogger {
Class<? extends RpcEndpoint> c;
- for (Class<? extends RpcEndpoint> rpcEndpoint :classes){
+ mainloop:
+ for (Class<? extends RpcEndpoint> rpcEndpoint : classes) {
c = rpcEndpoint;
- Class<?> rpcGatewayType = ReflectionUtil.getTemplateType1(c);
+ LOG.debug("-------------");
+ LOG.debug("c: {}", c);
- if (rpcGatewayType != null) {
- checkCompleteness(rpcEndpoint, (Class<? extends RpcGateway>) rpcGatewayType);
- } else {
- fail("Could not retrieve the rpc gateway class for the given rpc endpoint class " + rpcEndpoint.getName());
+ // skip abstract classes
+ if (Modifier.isAbstract(c.getModifiers())) {
+ LOG.debug("Skipping abstract class");
+ continue;
}
+
+ // check for type parameter bound to RpcGateway
+ // skip if one is found because a subclass will provide the concrete argument
+ TypeVariable<? extends Class<? extends RpcEndpoint>>[] typeParameters = c.getTypeParameters();
+ LOG.debug("Checking {} parameters.", typeParameters.length);
+ for (int i = 0; i < typeParameters.length; i++) {
+ for (Type bound : typeParameters[i].getBounds()) {
+ LOG.debug("checking bound {} of type parameter {}", bound, typeParameters[i]);
+ if (bound.toString().equals("interface " + RpcGateway.class.getName())) {
+ if (i > 0) {
+ fail("Type parameter for RpcGateway should come first in " + c);
+ }
+ LOG.debug("Skipping class with type parameter bound to RpcGateway.");
+ // Type parameter is bound to RpcGateway which a subclass will provide
+ continue mainloop;
+ }
+ }
+ }
+
+ // check if this class or any super class contains the RpcGateway argument
+ Class<?> rpcGatewayType;
+ do {
+ LOG.debug("checking type argument of class: {}", c);
+ rpcGatewayType = ReflectionUtil.getTemplateType1(c);
+ LOG.debug("type argument is: {}", rpcGatewayType);
+
+ c = (Class<? extends RpcEndpoint>) c.getSuperclass();
+
+ } while (!RpcGateway.class.isAssignableFrom(rpcGatewayType));
+
+ LOG.debug("Checking RRC completeness of endpoint '{}' with gateway '{}'",
+ rpcEndpoint.getSimpleName(), rpcGatewayType.getSimpleName());
+
+ checkCompleteness(rpcEndpoint, (Class<? extends RpcGateway>) rpcGatewayType);
}
}
@@ -352,7 +418,7 @@ public class RpcCompletenessTest extends TestLogger {
*/
private List<Method> getRpcMethodsFromGateway(Class<? extends RpcGateway> interfaceClass) {
if(!interfaceClass.isInterface()) {
- fail(interfaceClass.getName() + "is not a interface");
+ fail(interfaceClass.getName() + " is not a interface");
}
ArrayList<Method> allMethods = new ArrayList<>();
[37/50] [abbrv] flink git commit: [FLINK-4839] [cluster management]
JobManager handle TaskManager's slot offering
Posted by tr...@apache.org.
[FLINK-4839] [cluster management] JobManager handle TaskManager's slot offering
This closes #2647 #2643.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8ab27c0a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8ab27c0a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8ab27c0a
Branch: refs/heads/flip-6
Commit: 8ab27c0a6107dd30a858b3f38f53f2611ae94291
Parents: e9abc8d
Author: Kurt Young <yk...@gmail.com>
Authored: Mon Oct 17 18:15:26 2016 +0800
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:33 2016 +0100
----------------------------------------------------------------------
.../apache/flink/runtime/instance/SlotPool.java | 46 +++++++++---
.../flink/runtime/jobmaster/JobMaster.java | 50 ++++++++++++-
.../runtime/jobmaster/JobMasterGateway.java | 26 +++++--
.../resourcemanager/ResourceManager.java | 2 +-
.../runtime/taskexecutor/TaskExecutor.java | 34 +++++----
.../runtime/taskexecutor/slot/SlotOffer.java | 79 ++++++++++++++++++++
.../runtime/taskexecutor/slot/TaskSlot.java | 13 ++++
.../taskexecutor/slot/TaskSlotTable.java | 12 +--
.../runtime/taskexecutor/TaskExecutorTest.java | 16 +++-
9 files changed, 231 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8ab27c0a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
index de952c3..02166a4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
@@ -267,7 +267,7 @@ public class SlotPool implements SlotOwner {
}
// ------------------------------------------------------------------------
- // Slot De-allocation
+ // Slot releasing & offering
// ------------------------------------------------------------------------
/**
@@ -323,10 +323,6 @@ public class SlotPool implements SlotOwner {
return null;
}
- // ------------------------------------------------------------------------
- // Slot Releasing
- // ------------------------------------------------------------------------
-
/**
* Release slot to TaskManager, called for finished tasks or canceled jobs.
*
@@ -340,10 +336,6 @@ public class SlotPool implements SlotOwner {
}
}
- // ------------------------------------------------------------------------
- // Slot Offering
- // ------------------------------------------------------------------------
-
/**
* Slot offering by TaskManager with AllocationID. The AllocationID is originally generated by this pool and
* transfer through the ResourceManager to TaskManager. We use it to distinguish the different allocation
@@ -401,6 +393,39 @@ public class SlotPool implements SlotOwner {
}
// ------------------------------------------------------------------------
+ // Error Handling
+ // ------------------------------------------------------------------------
+
+ /**
+ * Fail the specified allocation and release the corresponding slot if we have one.
+ * This may triggered by JobManager when some slot allocation failed with timeout.
+ * Or this could be triggered by TaskManager, when it finds out something went wrong with the slot,
+ * and decided to take it back.
+ *
+ * @param allocationID Represents the allocation which should be failed
+ * @param cause The cause of the failure
+ */
+ public void failAllocation(final AllocationID allocationID, final Exception cause) {
+ synchronized (lock) {
+ // 1. check whether the allocation still pending
+ Tuple2<SlotRequest, FlinkCompletableFuture<SlotDescriptor>> pendingRequest =
+ pendingRequests.get(allocationID);
+ if (pendingRequest != null) {
+ pendingRequest.f1.completeExceptionally(cause);
+ return;
+ }
+
+ // 2. check whether we have a free slot corresponding to this allocation id
+ // TODO: add allocation id to slot descriptor, so we can remove it by allocation id
+
+ // 3. check whether we have a in-use slot corresponding to this allocation id
+ // TODO: needs mechanism to release the in-use Slot but don't return it back to this pool
+
+ // TODO: add some unit tests when the previous two are ready, the allocation may failed at any phase
+ }
+ }
+
+ // ------------------------------------------------------------------------
// Resource
// ------------------------------------------------------------------------
@@ -464,12 +489,13 @@ public class SlotPool implements SlotOwner {
*/
static class AllocatedSlots {
- /** All allocated slots organized by TaskManager */
+ /** All allocated slots organized by TaskManager's id */
private final Map<ResourceID, Set<Slot>> allocatedSlotsByResource;
/** All allocated slots organized by Slot object */
private final Map<Slot, AllocationID> allocatedSlots;
+ /** All allocated slot descriptors organized by Slot object */
private final Map<Slot, SlotDescriptor> allocatedSlotsWithDescriptor;
/** All allocated slots organized by AllocationID */
http://git-wip-us.apache.org/repos/asf/flink/blob/8ab27c0a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 306a28a..56fa3e7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -52,6 +52,7 @@ import org.apache.flink.runtime.executiongraph.restart.RestartStrategy;
import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.instance.Slot;
+import org.apache.flink.runtime.instance.SlotDescriptor;
import org.apache.flink.runtime.instance.SlotPool;
import org.apache.flink.runtime.io.network.PartitionState;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
@@ -85,6 +86,7 @@ import org.apache.flink.runtime.rpc.StartStoppable;
import org.apache.flink.runtime.state.CheckpointStateHandles;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
+import org.apache.flink.runtime.taskexecutor.slot.SlotOffer;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.runtime.util.SerializedThrowable;
@@ -95,7 +97,9 @@ import org.slf4j.Logger;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.Executor;
@@ -669,13 +673,51 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
}
@RpcMethod
- public Iterable<AllocationID> offerSlots(final Iterable<AllocationID> slots, UUID leaderId) {
- throw new UnsupportedOperationException("Has to be implemented.");
+ public Iterable<SlotOffer> offerSlots(final ResourceID taskManagerId,
+ final Iterable<SlotOffer> slots, final UUID leaderId) throws Exception
+ {
+ if (!this.leaderSessionID.equals(leaderId)) {
+ throw new Exception("Leader id not match, expected: " + this.leaderSessionID
+ + ", actual: " + leaderId);
+ }
+
+ Tuple2<TaskManagerLocation, TaskExecutorGateway> taskManager = registeredTaskManagers.get(taskManagerId);
+ if (taskManager == null) {
+ throw new Exception("Unknown TaskManager " + taskManagerId);
+ }
+
+ final Set<SlotOffer> acceptedSlotOffers = new HashSet<>(4);
+ for (SlotOffer slotOffer : slots) {
+ final SlotDescriptor slotDescriptor = new SlotDescriptor(
+ jobGraph.getJobID(),
+ taskManager.f0,
+ slotOffer.getSlotIndex(),
+ slotOffer.getResourceProfile(),
+ null); // TODO: replace the actor gateway with the new rpc gateway, it's ready (taskManager.f1)
+ if (slotPool.offerSlot(slotOffer.getAllocationId(), slotDescriptor)) {
+ acceptedSlotOffers.add(slotOffer);
+ }
+ }
+
+ return acceptedSlotOffers;
}
@RpcMethod
- public void failSlot(final AllocationID allocationId, UUID leaderId, Exception cause) {
- throw new UnsupportedOperationException("Has to be implemented.");
+ public void failSlot(final ResourceID taskManagerId,
+ final AllocationID allocationId,
+ final UUID leaderId,
+ final Exception cause) throws Exception
+ {
+ if (!this.leaderSessionID.equals(leaderId)) {
+ throw new Exception("Leader id not match, expected: " + this.leaderSessionID
+ + ", actual: " + leaderId);
+ }
+
+ if (!registeredTaskManagers.containsKey(taskManagerId)) {
+ throw new Exception("Unknown TaskManager " + taskManagerId);
+ }
+
+ slotPool.failAllocation(allocationId, cause);
}
@RpcMethod
http://git-wip-us.apache.org/repos/asf/flink/blob/8ab27c0a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
index 4ee9f92..27308d3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
@@ -40,6 +40,7 @@ import org.apache.flink.runtime.registration.RegistrationResponse;
import org.apache.flink.runtime.rpc.RpcTimeout;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.KvState;
+import org.apache.flink.runtime.taskexecutor.slot.SlotOffer;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
@@ -177,21 +178,30 @@ public interface JobMasterGateway extends CheckpointCoordinatorGateway {
/**
* Offer the given slots to the job manager. The response contains the set of accepted slots.
*
- * @param slots to offer to the job manager
- * @param leaderId identifying the job leader
- * @param timeout for the rpc call
+ * @param taskManagerId identifying the task manager
+ * @param slots to offer to the job manager
+ * @param leaderId identifying the job leader
+ * @param timeout for the rpc call
* @return Future set of accepted slots.
*/
- Future<Iterable<AllocationID>> offerSlots(final Iterable<AllocationID> slots, UUID leaderId, @RpcTimeout final Time timeout);
+ Future<Iterable<SlotOffer>> offerSlots(
+ final ResourceID taskManagerId,
+ final Iterable<SlotOffer> slots,
+ final UUID leaderId,
+ @RpcTimeout final Time timeout);
/**
* Fail the slot with the given allocation id and cause.
*
- * @param allocationId identifying the slot to fail
- * @param leaderId identifying the job leader
- * @param cause of the failing
+ * @param taskManagerId identifying the task manager
+ * @param allocationId identifying the slot to fail
+ * @param leaderId identifying the job leader
+ * @param cause of the failing
*/
- void failSlot(final AllocationID allocationId, UUID leaderId, Exception cause);
+ void failSlot(final ResourceID taskManagerId,
+ final AllocationID allocationId,
+ final UUID leaderId,
+ final Exception cause);
/**
* Register the task manager at the job manager.
http://git-wip-us.apache.org/repos/asf/flink/blob/8ab27c0a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index 3122804..f1a5073 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -69,7 +69,7 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
* ResourceManager implementation. The resource manager is responsible for resource de-/allocation
* and bookkeeping.
*
- * It offers the following methods as part of its rpc interface to interact with the him remotely:
+ * It offers the following methods as part of its rpc interface to interact with him remotely:
* <ul>
* <li>{@link #registerJobMaster(UUID, UUID, String, JobID)} registers a {@link JobMaster} at the resource manager</li>
* <li>{@link #requestSlot(UUID, UUID, SlotRequest)} requests a slot from the resource manager</li>
http://git-wip-us.apache.org/repos/asf/flink/blob/8ab27c0a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index 3e3a544..601d804 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -70,6 +70,8 @@ import org.apache.flink.runtime.taskexecutor.rpc.RpcResultPartitionConsumableNot
import org.apache.flink.runtime.taskexecutor.slot.SlotActions;
import org.apache.flink.runtime.taskexecutor.slot.SlotNotActiveException;
import org.apache.flink.runtime.taskexecutor.slot.SlotNotFoundException;
+import org.apache.flink.runtime.taskexecutor.slot.SlotOffer;
+import org.apache.flink.runtime.taskexecutor.slot.TaskSlot;
import org.apache.flink.runtime.taskexecutor.slot.TaskSlotTable;
import org.apache.flink.runtime.taskmanager.CheckpointResponder;
import org.apache.flink.runtime.taskmanager.Task;
@@ -630,47 +632,49 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
final JobMasterGateway jobMasterGateway = jobManagerConnection.getJobManagerGateway();
- final Iterator<AllocationID> reservedSlotsIterator = taskSlotTable.getAllocatedSlots(jobId);
+ final Iterator<TaskSlot> reservedSlotsIterator = taskSlotTable.getAllocatedSlots(jobId);
final UUID leaderId = jobManagerConnection.getLeaderId();
- final Collection<AllocationID> reservedSlots = new HashSet<>(2);
+ final Collection<SlotOffer> reservedSlots = new HashSet<>(2);
while (reservedSlotsIterator.hasNext()) {
- reservedSlots.add(reservedSlotsIterator.next());
+ reservedSlots.add(reservedSlotsIterator.next().generateSlotOffer());
}
- Future<Iterable<AllocationID>> acceptedSlotsFuture = jobMasterGateway.offerSlots(
+ Future<Iterable<SlotOffer>> acceptedSlotsFuture = jobMasterGateway.offerSlots(
+ getResourceID(),
reservedSlots,
leaderId,
taskManagerConfiguration.getTimeout());
- acceptedSlotsFuture.thenAcceptAsync(new AcceptFunction<Iterable<AllocationID>>() {
+ acceptedSlotsFuture.thenAcceptAsync(new AcceptFunction<Iterable<SlotOffer>>() {
@Override
- public void accept(Iterable<AllocationID> acceptedSlots) {
+ public void accept(Iterable<SlotOffer> acceptedSlots) {
// check if the response is still valid
if (isJobManagerConnectionValid(jobId, leaderId)) {
// mark accepted slots active
- for (AllocationID acceptedSlot: acceptedSlots) {
+ for (SlotOffer acceptedSlot: acceptedSlots) {
try {
- if (!taskSlotTable.markSlotActive(acceptedSlot)) {
+ if (!taskSlotTable.markSlotActive(acceptedSlot.getAllocationId())) {
// the slot is either free or releasing at the moment
final String message = "Could not mark slot " + jobId + " active.";
log.debug(message);
- jobMasterGateway.failSlot(acceptedSlot, leaderId, new Exception(message));
+ jobMasterGateway.failSlot(getResourceID(), acceptedSlot.getAllocationId(),
+ leaderId, new Exception(message));
}
// remove the assigned slots so that we can free the left overs
reservedSlots.remove(acceptedSlot);
} catch (SlotNotFoundException e) {
log.debug("Could not mark slot {} active.", acceptedSlot, e);
- jobMasterGateway.failSlot(acceptedSlot, leaderId, e);
+ jobMasterGateway.failSlot(getResourceID(), acceptedSlot.getAllocationId(), leaderId, e);
}
}
final Exception e = new Exception("The slot was rejected by the JobManager.");
- for (AllocationID rejectedSlot: reservedSlots) {
- freeSlot(rejectedSlot, e);
+ for (SlotOffer rejectedSlot: reservedSlots) {
+ freeSlot(rejectedSlot.getAllocationId(), e);
}
} else {
// discard the response since there is a new leader for the job
@@ -688,8 +692,8 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
offerSlotsToJobManager(jobId);
} else {
// We encountered an exception. Free the slots and return them to the RM.
- for (AllocationID reservedSlot: reservedSlots) {
- freeSlot(reservedSlot, throwable);
+ for (SlotOffer reservedSlot: reservedSlots) {
+ freeSlot(reservedSlot.getAllocationId(), throwable);
}
}
@@ -840,7 +844,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
private void unregisterTaskAndNotifyFinalState(
final UUID jobMasterLeaderId,
- final JobMasterGateway jobMasterGateway,
+ final JobMasterGateway jobMasterGateway,
final ExecutionAttemptID executionAttemptID) {
Task task = taskSlotTable.removeTask(executionAttemptID);
http://git-wip-us.apache.org/repos/asf/flink/blob/8ab27c0a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotOffer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotOffer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotOffer.java
new file mode 100644
index 0000000..f8d7e6c
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotOffer.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.slot;
+
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import org.apache.flink.util.Preconditions;
+
+import java.io.Serializable;
+
+/**
+ * Describe the slot offering to job manager provided by task manager.
+ */
+public class SlotOffer implements Serializable {
+
+ private static final long serialVersionUID = -7067814231108250971L;
+
+ /** Allocation id of this slot, this would be the only identifier for this slot offer */
+ private AllocationID allocationId;
+
+ /** Index of the offered slot */
+ private final int slotIndex;
+
+ /** The resource profile of the offered slot */
+ private final ResourceProfile resourceProfile;
+
+ public SlotOffer(final AllocationID allocationID, final int index, final ResourceProfile resourceProfile) {
+ Preconditions.checkArgument(0 <= index, "The index must be greater than 0.");
+ this.allocationId = Preconditions.checkNotNull(allocationID);
+ this.slotIndex = index;
+ this.resourceProfile = Preconditions.checkNotNull(resourceProfile);
+ }
+
+ public AllocationID getAllocationId() {
+ return allocationId;
+ }
+
+ public int getSlotIndex() {
+ return slotIndex;
+ }
+
+ public ResourceProfile getResourceProfile() {
+ return resourceProfile;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ SlotOffer slotOffer = (SlotOffer) o;
+ return allocationId.equals(slotOffer.allocationId);
+ }
+
+ @Override
+ public int hashCode() {
+ return allocationId.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/8ab27c0a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlot.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlot.java
index 0942772..e12c15b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlot.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlot.java
@@ -286,4 +286,17 @@ public class TaskSlot {
state = TaskSlotState.RELEASING;
return true;
}
+
+ /**
+ * Generate the slot offer from this TaskSlot.
+ *
+ * @return The sot offer which this task slot can provide
+ */
+ public SlotOffer generateSlotOffer() {
+ Preconditions.checkState(TaskSlotState.ACTIVE == state || TaskSlotState.ALLOCATED == state,
+ "The task slot is not in state active or allocated.");
+ Preconditions.checkState(allocationId != null, "The task slot are not allocated");
+
+ return new SlotOffer(allocationId, index, resourceProfile);
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8ab27c0a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
index 88123b4..88b83a0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
@@ -70,7 +70,7 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
/** Interface for slot actions, such as freeing them or timing them out */
private SlotActions slotActions;
-
+
/** Whether the table has been started */
private boolean started;
@@ -250,7 +250,7 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
*/
public int freeSlot(AllocationID allocationId, Throwable cause) throws SlotNotFoundException {
checkInit();
-
+
if (LOG.isDebugEnabled()) {
LOG.debug("Free slot {}.", allocationId, cause);
} else {
@@ -370,13 +370,13 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
}
/**
- * Return an iterator of allocated slots (their allocation ids) for the given job id.
+ * Return an iterator of allocated slots for the given job id.
*
* @param jobId for which to return the allocated slots
- * @return Iterator of allocation ids of allocated slots.
+ * @return Iterator of allocated slots.
*/
- public Iterator<AllocationID> getAllocatedSlots(JobID jobId) {
- return new AllocationIDIterator(jobId, TaskSlotState.ALLOCATED);
+ public Iterator<TaskSlot> getAllocatedSlots(JobID jobId) {
+ return new TaskSlotIterator(jobId, TaskSlotState.ALLOCATED);
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/8ab27c0a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index 2b5d2dd..87bde35 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -63,6 +63,7 @@ import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequ
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.TestingSerialRpcService;
+import org.apache.flink.runtime.taskexecutor.slot.SlotOffer;
import org.apache.flink.runtime.taskexecutor.slot.TaskSlotTable;
import org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException;
import org.apache.flink.runtime.taskexecutor.slot.TimerService;
@@ -391,6 +392,7 @@ public class TaskExecutorTest extends TestLogger {
final AllocationID allocationId = new AllocationID();
final SlotID slotId = new SlotID(resourceId, 0);
+ final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.UNKNOWN);
try {
TaskExecutor taskManager = new TaskExecutor(
@@ -425,7 +427,11 @@ public class TaskExecutorTest extends TestLogger {
jobManagerLeaderRetrievalService.notifyListener(jobManagerAddress, jobManagerLeaderId);
// the job leader should get the allocation id offered
- verify(jobMasterGateway).offerSlots((Iterable<AllocationID>)Matchers.argThat(contains(allocationId)), eq(jobManagerLeaderId), any(Time.class));
+ verify(jobMasterGateway).offerSlots(
+ any(ResourceID.class),
+ (Iterable<SlotOffer>)Matchers.argThat(contains(slotOffer)),
+ eq(jobManagerLeaderId),
+ any(Time.class));
} finally {
// check if a concurrent error occurred
testingFatalErrorHandler.rethrowException();
@@ -481,6 +487,9 @@ public class TaskExecutorTest extends TestLogger {
final AllocationID allocationId1 = new AllocationID();
final AllocationID allocationId2 = new AllocationID();
+ final SlotOffer offer1 = new SlotOffer(allocationId1, 0, ResourceProfile.UNKNOWN);
+ final SlotOffer offer2 = new SlotOffer(allocationId2, 0, ResourceProfile.UNKNOWN);
+
final JobMasterGateway jobMasterGateway = mock(JobMasterGateway.class);
when(jobMasterGateway.registerTaskManager(
@@ -491,8 +500,9 @@ public class TaskExecutorTest extends TestLogger {
)).thenReturn(FlinkCompletableFuture.<RegistrationResponse>completed(new JMTMRegistrationSuccess(jmResourceId, blobPort)));
when(jobMasterGateway.getAddress()).thenReturn(jobManagerAddress);
- when(jobMasterGateway.offerSlots(any(Iterable.class), eq(jobManagerLeaderId), any(Time.class)))
- .thenReturn(FlinkCompletableFuture.completed((Iterable<AllocationID>)Collections.singleton(allocationId1)));
+ when(jobMasterGateway.offerSlots(
+ any(ResourceID.class), any(Iterable.class), eq(jobManagerLeaderId), any(Time.class)))
+ .thenReturn(FlinkCompletableFuture.completed((Iterable<SlotOffer>)Collections.singleton(offer1)));
rpc.registerGateway(resourceManagerAddress, resourceManagerGateway);
rpc.registerGateway(jobManagerAddress, jobMasterGateway);
[07/50] [abbrv] flink git commit: [hotfix] Replace
TaskManager.createTaskManagerComponents by TaskManagerServices
Posted by tr...@apache.org.
[hotfix] Replace TaskManager.createTaskManagerComponents by TaskManagerServices
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8e653397
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8e653397
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8e653397
Branch: refs/heads/flip-6
Commit: 8e653397a931497026c8ae8be05b48a108a58edf
Parents: 5219b40
Author: Till Rohrmann <tr...@apache.org>
Authored: Wed Sep 28 14:04:54 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:28 2016 +0100
----------------------------------------------------------------------
.../clusterframework/MesosTaskManager.scala | 3 +-
.../taskexecutor/TaskManagerConfiguration.java | 25 +-
.../TaskManagerServicesConfiguration.java | 2 +-
.../minicluster/LocalFlinkMiniCluster.scala | 47 +-
.../flink/runtime/taskmanager/TaskManager.scala | 601 ++-----------------
.../taskmanager/TaskManagerConfiguration.scala | 56 --
...askManagerComponentsStartupShutdownTest.java | 24 +-
.../testingUtils/TestingTaskManager.scala | 3 +-
.../runtime/testingUtils/TestingUtils.scala | 1 -
.../flink/yarn/TestingYarnTaskManager.scala | 3 +-
.../org/apache/flink/yarn/YarnTaskManager.scala | 3 +-
11 files changed, 126 insertions(+), 642 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8e653397/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManager.scala b/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManager.scala
index 3972a57..e8d6a58 100644
--- a/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManager.scala
+++ b/flink-mesos/src/main/scala/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManager.scala
@@ -24,7 +24,8 @@ import org.apache.flink.runtime.io.network.NetworkEnvironment
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService
import org.apache.flink.runtime.memory.MemoryManager
import org.apache.flink.runtime.metrics.MetricRegistry
-import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerConfiguration, TaskManagerLocation}
+import org.apache.flink.runtime.taskexecutor.TaskManagerConfiguration
+import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerLocation}
/** An extension of the TaskManager that listens for additional Mesos-related
* messages.
http://git-wip-us.apache.org/repos/asf/flink/blob/8e653397/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
index 32eb8c1..f58af77 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
@@ -41,6 +41,7 @@ public class TaskManagerConfiguration {
private final String[] tmpDirPaths;
private final Time timeout;
+ // null indicates an infinite duration
private final Time maxRegistrationDuration;
private final Time initialRegistrationPause;
private final Time maxRegistrationPause;
@@ -48,6 +49,9 @@ public class TaskManagerConfiguration {
private final long cleanupInterval;
+ // TODO: remove necessity for complete configuration object
+ private final Configuration configuration;
+
public TaskManagerConfiguration(
int numberSlots,
String[] tmpDirPaths,
@@ -56,16 +60,18 @@ public class TaskManagerConfiguration {
Time initialRegistrationPause,
Time maxRegistrationPause,
Time refusedRegistrationPause,
- long cleanupInterval) {
+ long cleanupInterval,
+ Configuration configuration) {
this.numberSlots = numberSlots;
this.tmpDirPaths = Preconditions.checkNotNull(tmpDirPaths);
this.timeout = Preconditions.checkNotNull(timeout);
- this.maxRegistrationDuration = Preconditions.checkNotNull(maxRegistrationDuration);
+ this.maxRegistrationDuration = maxRegistrationDuration;
this.initialRegistrationPause = Preconditions.checkNotNull(initialRegistrationPause);
this.maxRegistrationPause = Preconditions.checkNotNull(maxRegistrationPause);
this.refusedRegistrationPause = Preconditions.checkNotNull(refusedRegistrationPause);
this.cleanupInterval = Preconditions.checkNotNull(cleanupInterval);
+ this.configuration = Preconditions.checkNotNull(configuration);
}
public int getNumberSlots() {
@@ -100,6 +106,10 @@ public class TaskManagerConfiguration {
return cleanupInterval;
}
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
// --------------------------------------------------------------------------------------------
// Static factory methods
// --------------------------------------------------------------------------------------------
@@ -138,7 +148,7 @@ public class TaskManagerConfiguration {
ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION,
ConfigConstants.DEFAULT_TASK_MANAGER_MAX_REGISTRATION_DURATION));
if (maxRegistrationDuration.isFinite()) {
- finiteRegistrationDuration = Time.seconds(maxRegistrationDuration.toSeconds());
+ finiteRegistrationDuration = Time.milliseconds(maxRegistrationDuration.toMillis());
} else {
finiteRegistrationDuration = null;
}
@@ -153,7 +163,7 @@ public class TaskManagerConfiguration {
ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE,
ConfigConstants.DEFAULT_TASK_MANAGER_INITIAL_REGISTRATION_PAUSE));
if (pause.isFinite()) {
- initialRegistrationPause = Time.seconds(pause.toSeconds());
+ initialRegistrationPause = Time.milliseconds(pause.toMillis());
} else {
throw new IllegalArgumentException("The initial registration pause must be finite: " + pause);
}
@@ -168,7 +178,7 @@ public class TaskManagerConfiguration {
ConfigConstants.TASK_MANAGER_MAX_REGISTARTION_PAUSE,
ConfigConstants.DEFAULT_TASK_MANAGER_MAX_REGISTRATION_PAUSE));
if (pause.isFinite()) {
- maxRegistrationPause = Time.seconds(pause.toSeconds());
+ maxRegistrationPause = Time.milliseconds(pause.toMillis());
} else {
throw new IllegalArgumentException("The maximum registration pause must be finite: " + pause);
}
@@ -183,7 +193,7 @@ public class TaskManagerConfiguration {
ConfigConstants.TASK_MANAGER_REFUSED_REGISTRATION_PAUSE,
ConfigConstants.DEFAULT_TASK_MANAGER_REFUSED_REGISTRATION_PAUSE));
if (pause.isFinite()) {
- refusedRegistrationPause = Time.seconds(pause.toSeconds());
+ refusedRegistrationPause = Time.milliseconds(pause.toMillis());
} else {
throw new IllegalArgumentException("The refused registration pause must be finite: " + pause);
}
@@ -200,6 +210,7 @@ public class TaskManagerConfiguration {
initialRegistrationPause,
maxRegistrationPause,
refusedRegistrationPause,
- cleanupInterval);
+ cleanupInterval,
+ configuration);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8e653397/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
index 66d969a..80dfc09 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
@@ -208,7 +208,7 @@ public class TaskManagerServicesConfiguration {
int dataport = configuration.getInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
ConfigConstants.DEFAULT_TASK_MANAGER_DATA_PORT);
- checkConfigParameter(dataport > 0, dataport, ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
+ checkConfigParameter(dataport >= 0, dataport, ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
"Leave config parameter empty or use 0 to let the system choose a port automatically.");
checkConfigParameter(slots >= 1, slots, ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS,
http://git-wip-us.apache.org/repos/asf/flink/blob/8e653397/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
index cad2648..eac0a51 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.minicluster
+import java.net.InetAddress
import java.util.concurrent.ExecutorService
import akka.actor.{ActorRef, ActorSystem, Props}
@@ -42,8 +43,9 @@ import org.apache.flink.runtime.memory.MemoryManager
import org.apache.flink.runtime.messages.JobManagerMessages
import org.apache.flink.runtime.messages.JobManagerMessages.{RunningJobsStatus, StoppingFailure, StoppingResponse}
import org.apache.flink.runtime.metrics.MetricRegistry
-import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerConfiguration, TaskManagerLocation}
-import org.apache.flink.runtime.util.EnvironmentInformation
+import org.apache.flink.runtime.taskexecutor.{TaskManagerConfiguration, TaskManagerServices, TaskManagerServicesConfiguration}
+import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerLocation}
+import org.apache.flink.runtime.util.{EnvironmentInformation, LeaderRetrievalUtils}
import scala.concurrent.Await
import scala.concurrent.duration.FiniteDuration
@@ -195,31 +197,32 @@ class LocalFlinkMiniCluster(
val resourceID = ResourceID.generate() // generate random resource id
- val (taskManagerConfig,
- taskManagerLocation,
- memoryManager,
- ioManager,
- network,
- leaderRetrievalService,
- metricsRegistry) = TaskManager.createTaskManagerComponents(
+ val taskManagerAddress = InetAddress.getByName(hostname)
+
+ val taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(config)
+ val taskManagerServicesConfiguration = TaskManagerServicesConfiguration.fromConfiguration(
config,
- resourceID,
- hostname, // network interface to bind to
- localExecution, // start network stack?
- Some(createLeaderRetrievalService()))
+ taskManagerAddress,
+ localExecution)
+
+ val taskManagerServices = TaskManagerServices.fromConfiguration(
+ taskManagerServicesConfiguration,
+ resourceID)
+
+ val metricRegistry = taskManagerServices.getMetricRegistry()
val props = getTaskManagerProps(
taskManagerClass,
- taskManagerConfig,
+ taskManagerConfiguration,
resourceID,
- taskManagerLocation,
- memoryManager,
- ioManager,
- network,
- leaderRetrievalService,
- metricsRegistry)
-
- metricsRegistry.startQueryService(system, resourceID)
+ taskManagerServices.getTaskManagerLocation(),
+ taskManagerServices.getMemoryManager(),
+ taskManagerServices.getIOManager(),
+ taskManagerServices.getNetworkEnvironment,
+ createLeaderRetrievalService(),
+ metricRegistry)
+
+ metricRegistry.startQueryService(system, resourceID)
system.actorOf(props, taskManagerActorName)
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8e653397/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index c789156..0701f3a 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -31,7 +31,6 @@ import grizzled.slf4j.Logger
import org.apache.commons.lang3.exception.ExceptionUtils
import org.apache.flink.configuration._
import org.apache.flink.core.fs.FileSystem
-import org.apache.flink.core.memory.{HeapMemorySegment, HybridMemorySegment, MemorySegmentFactory, MemoryType}
import org.apache.flink.runtime.accumulators.AccumulatorSnapshot
import org.apache.flink.runtime.clusterframework.messages.StopCluster
import org.apache.flink.runtime.clusterframework.types.ResourceID
@@ -44,12 +43,8 @@ import org.apache.flink.runtime.execution.librarycache.{BlobLibraryCacheManager,
import org.apache.flink.runtime.executiongraph.{ExecutionAttemptID, PartitionInfo}
import org.apache.flink.runtime.filecache.FileCache
import org.apache.flink.runtime.instance.{AkkaActorGateway, HardwareDescription, InstanceID}
-import org.apache.flink.runtime.io.disk.iomanager.IOManager.IOMode
-import org.apache.flink.runtime.io.disk.iomanager.{IOManager, IOManagerAsync}
-import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool
-import org.apache.flink.runtime.io.network.{LocalConnectionManager, NetworkEnvironment, TaskEventDispatcher}
-import org.apache.flink.runtime.io.network.netty.{NettyConfig, NettyConnectionManager, PartitionStateChecker}
-import org.apache.flink.runtime.io.network.partition.{ResultPartitionConsumableNotifier, ResultPartitionManager}
+import org.apache.flink.runtime.io.disk.iomanager.IOManager
+import org.apache.flink.runtime.io.network.NetworkEnvironment
import org.apache.flink.runtime.leaderretrieval.{LeaderRetrievalListener, LeaderRetrievalService}
import org.apache.flink.runtime.memory.MemoryManager
import org.apache.flink.runtime.messages.{Acknowledge, StackTraceSampleResponse}
@@ -59,17 +54,16 @@ import org.apache.flink.runtime.messages.StackTraceSampleMessages.{SampleTaskSta
import org.apache.flink.runtime.messages.TaskManagerMessages._
import org.apache.flink.runtime.messages.TaskMessages._
import org.apache.flink.runtime.messages.checkpoint.{AbstractCheckpointMessage, NotifyCheckpointComplete, TriggerCheckpoint}
-import org.apache.flink.runtime.metrics.{MetricRegistryConfiguration, MetricRegistry => FlinkMetricRegistry}
+import org.apache.flink.runtime.metrics.{MetricRegistry => FlinkMetricRegistry}
import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup
import org.apache.flink.runtime.metrics.util.MetricUtils
import org.apache.flink.runtime.process.ProcessReaper
-import org.apache.flink.runtime.query.KvStateRegistry
-import org.apache.flink.runtime.query.netty.{DisabledKvStateRequestStats, KvStateServer}
import org.apache.flink.runtime.security.SecurityContext.{FlinkSecuredRunner, SecurityConfiguration}
import org.apache.flink.runtime.security.SecurityContext
+import org.apache.flink.runtime.taskexecutor.{TaskManagerConfiguration, TaskManagerServices, TaskManagerServicesConfiguration}
import org.apache.flink.runtime.util._
import org.apache.flink.runtime.{FlinkActor, LeaderSessionMessageFilter, LogMessages}
-import org.apache.flink.util.{MathUtils, NetUtils}
+import org.apache.flink.util.NetUtils
import scala.collection.JavaConverters._
import scala.concurrent._
@@ -136,7 +130,7 @@ class TaskManager(
override val log = Logger(getClass)
/** The timeout for all actor ask futures */
- protected val askTimeout = new Timeout(config.timeout)
+ protected val askTimeout = new Timeout(config.getTimeout().getSize, config.getTimeout().getUnit())
/** The TaskManager's physical execution resources */
protected val resources = HardwareDescription.extractFromSystem(memoryManager.getMemorySize())
@@ -148,7 +142,7 @@ class TaskManager(
protected val bcVarManager = new BroadcastVariableManager()
/** Handler for distributed files cached by this TaskManager */
- protected val fileCache = new FileCache(config.configuration)
+ protected val fileCache = new FileCache(config.getConfiguration())
private var taskManagerMetricGroup : TaskManagerMetricGroup = _
@@ -172,8 +166,8 @@ class TaskManager(
private val runtimeInfo = new TaskManagerRuntimeInfo(
location.getHostname(),
- new UnmodifiableConfiguration(config.configuration),
- config.tmpDirPaths)
+ new UnmodifiableConfiguration(config.getConfiguration()),
+ config.getTmpDirPaths())
private var scheduledTaskManagerRegistration: Option[Cancellable] = None
private var currentRegistrationRun: UUID = UUID.randomUUID()
@@ -587,7 +581,9 @@ class TaskManager(
)
// the next timeout computes via exponential backoff with cap
- val nextTimeout = (timeout * 2).min(config.maxRegistrationPause)
+ val nextTimeout = (timeout * 2).min(new FiniteDuration(
+ config.getMaxRegistrationPause().toMilliseconds,
+ TimeUnit.MILLISECONDS))
// schedule (with our timeout s delay) a check triggers a new registration
// attempt, if we are not registered by then
@@ -661,10 +657,14 @@ class TaskManager(
if(jobManagerAkkaURL.isDefined) {
// try the registration again after some time
- val delay: FiniteDuration = config.refusedRegistrationPause
- val deadline: Option[Deadline] = config.maxRegistrationDuration.map {
- timeout => timeout + delay fromNow
- }
+ val delay: FiniteDuration = new FiniteDuration(
+ config.getRefusedRegistrationPause().getSize(),
+ config.getRefusedRegistrationPause().getUnit())
+ val deadline: Option[Deadline] = Option(config.getMaxRegistrationDuration())
+ .map {
+ duration => new FiniteDuration(duration.getSize(), duration.getUnit()) +
+ delay fromNow
+ }
// start a new registration run
currentRegistrationRun = UUID.randomUUID()
@@ -676,7 +676,9 @@ class TaskManager(
self ! decorateMessage(
TriggerTaskManagerRegistration(
jobManagerAkkaURL.get,
- config.initialRegistrationPause,
+ new FiniteDuration(
+ config.getInitialRegistrationPause().getSize(),
+ config.getInitialRegistrationPause().getUnit()),
deadline,
1,
currentRegistrationRun)
@@ -817,7 +819,7 @@ class TaskManager(
requestType: LogTypeRequest,
jobManager: ActorRef)
: Unit = {
- val logFilePathOption = Option(config.configuration.getString(
+ val logFilePathOption = Option(config.getConfiguration().getString(
ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, System.getProperty("log.file")));
logFilePathOption match {
case None => throw new IOException("TaskManager log files are unavailable. " +
@@ -950,9 +952,10 @@ class TaskManager(
log.info(s"Determined BLOB server address to be $address. Starting BLOB cache.")
try {
- val blobcache = new BlobCache(address, config.configuration)
+ val blobcache = new BlobCache(address, config.getConfiguration())
blobService = Option(blobcache)
- libraryCacheManager = Some(new BlobLibraryCacheManager(blobcache, config.cleanupInterval))
+ libraryCacheManager = Some(
+ new BlobLibraryCacheManager(blobcache, config.getCleanupInterval()))
}
catch {
case e: Exception =>
@@ -1134,7 +1137,9 @@ class TaskManager(
tdd.getJobID,
tdd.getVertexID,
tdd.getExecutionId,
- config.timeout)
+ new FiniteDuration(
+ config.getTimeout().getSize(),
+ config.getTimeout().getUnit()))
val task = new Task(
tdd,
@@ -1408,7 +1413,8 @@ class TaskManager(
def triggerTaskManagerRegistration(): Unit = {
if(jobManagerAkkaURL.isDefined) {
// begin attempts to reconnect
- val deadline: Option[Deadline] = config.maxRegistrationDuration.map(_.fromNow)
+ val deadline: Option[Deadline] = Option(config.getMaxRegistrationDuration())
+ .map{ duration => new FiniteDuration(duration.getSize(), duration.getUnit()).fromNow }
// start a new registration run
currentRegistrationRun = UUID.randomUUID()
@@ -1418,7 +1424,9 @@ class TaskManager(
self ! decorateMessage(
TriggerTaskManagerRegistration(
jobManagerAkkaURL.get,
- config.initialRegistrationPause,
+ new FiniteDuration(
+ config.getInitialRegistrationPause().getSize(),
+ config.getInitialRegistrationPause().getUnit()),
deadline,
1,
currentRegistrationRun)
@@ -1825,32 +1833,37 @@ object TaskManager {
taskManagerClass: Class[_ <: TaskManager])
: ActorRef = {
- val (taskManagerConfig,
- connectionInfo,
- memoryManager,
- ioManager,
- network,
- leaderRetrievalService,
- metricsRegistry) = createTaskManagerComponents(
- configuration,
- resourceID,
- taskManagerHostname,
- localTaskManagerCommunication,
- leaderRetrievalServiceOption)
+ val taskManagerAddress = InetAddress.getByName(taskManagerHostname)
+
+ val taskManagerServicesConfiguration = TaskManagerServicesConfiguration
+ .fromConfiguration(configuration, taskManagerAddress, false)
+
+ val taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration)
+
+ val taskManagerServices = TaskManagerServices.fromConfiguration(
+ taskManagerServicesConfiguration,
+ resourceID)
+
+ val metricRegistry = taskManagerServices.getMetricRegistry()
+
+ val leaderRetrievalService = leaderRetrievalServiceOption match {
+ case Some(lrs) => lrs
+ case None => LeaderRetrievalUtils.createLeaderRetrievalService(configuration)
+ }
// create the actor properties (which define the actor constructor parameters)
val tmProps = getTaskManagerProps(
taskManagerClass,
- taskManagerConfig,
+ taskManagerConfiguration,
resourceID,
- connectionInfo,
- memoryManager,
- ioManager,
- network,
+ taskManagerServices.getTaskManagerLocation(),
+ taskManagerServices.getMemoryManager(),
+ taskManagerServices.getIOManager(),
+ taskManagerServices.getNetworkEnvironment(),
leaderRetrievalService,
- metricsRegistry)
+ metricRegistry)
- metricsRegistry.startQueryService(actorSystem, resourceID)
+ metricRegistry.startQueryService(actorSystem, resourceID)
taskManagerActorName match {
case Some(actorName) => actorSystem.actorOf(tmProps, actorName)
@@ -1877,211 +1890,11 @@ object TaskManager {
memoryManager,
ioManager,
networkEnvironment,
- taskManagerConfig.numberOfSlots,
+ taskManagerConfig.getNumberSlots(),
leaderRetrievalService,
metricsRegistry)
}
- def createTaskManagerComponents(
- configuration: Configuration,
- resourceID: ResourceID,
- taskManagerHostname: String,
- localTaskManagerCommunication: Boolean,
- leaderRetrievalServiceOption: Option[LeaderRetrievalService]):
- (TaskManagerConfiguration,
- TaskManagerLocation,
- MemoryManager,
- IOManager,
- NetworkEnvironment,
- LeaderRetrievalService,
- FlinkMetricRegistry) = {
-
- val (taskManagerConfig : TaskManagerConfiguration,
- netConfig: NetworkEnvironmentConfiguration,
- taskManagerAddress: InetSocketAddress,
- memType: MemoryType
- ) = parseTaskManagerConfiguration(
- configuration,
- taskManagerHostname,
- localTaskManagerCommunication)
-
- // pre-start checks
- checkTempDirs(taskManagerConfig.tmpDirPaths)
-
- val networkBufferPool = new NetworkBufferPool(
- netConfig.numNetworkBuffers,
- netConfig.networkBufferSize,
- netConfig.memoryType)
-
- val connectionManager = Option(netConfig.nettyConfig) match {
- case Some(nettyConfig) => new NettyConnectionManager(nettyConfig)
- case None => new LocalConnectionManager()
- }
-
- val resultPartitionManager = new ResultPartitionManager()
- val taskEventDispatcher = new TaskEventDispatcher()
-
- val kvStateRegistry = new KvStateRegistry()
-
- val kvStateServer = Option(netConfig.nettyConfig) match {
- case Some(nettyConfig) =>
-
- val numNetworkThreads = if (netConfig.queryServerNetworkThreads == 0) {
- nettyConfig.getNumberOfSlots
- } else {
- netConfig.queryServerNetworkThreads
- }
-
- val numQueryThreads = if (netConfig.queryServerQueryThreads == 0) {
- nettyConfig.getNumberOfSlots
- } else {
- netConfig.queryServerQueryThreads
- }
-
- new KvStateServer(
- taskManagerAddress.getAddress(),
- netConfig.queryServerPort,
- numNetworkThreads,
- numQueryThreads,
- kvStateRegistry,
- new DisabledKvStateRequestStats())
-
- case None => null
- }
-
- // we start the network first, to make sure it can allocate its buffers first
- val network = new NetworkEnvironment(
- networkBufferPool,
- connectionManager,
- resultPartitionManager,
- taskEventDispatcher,
- kvStateRegistry,
- kvStateServer,
- netConfig.ioMode,
- netConfig.partitionRequestInitialBackoff,
- netConfig.partitinRequestMaxBackoff)
-
- network.start()
-
- val taskManagerLocation = new TaskManagerLocation(
- resourceID,
- taskManagerAddress.getAddress(),
- network.getConnectionManager().getDataPort())
-
- // computing the amount of memory to use depends on how much memory is available
- // it strictly needs to happen AFTER the network stack has been initialized
-
- // check if a value has been configured
- val configuredMemory = configuration.getLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L)
- checkConfigParameter(configuredMemory == -1 || configuredMemory > 0, configuredMemory,
- ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY,
- "MemoryManager needs at least one MB of memory. " +
- "If you leave this config parameter empty, the system automatically " +
- "pick a fraction of the available memory.")
-
-
- val preAllocateMemory = configuration.getBoolean(
- ConfigConstants.TASK_MANAGER_MEMORY_PRE_ALLOCATE_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_PRE_ALLOCATE)
-
- val memorySize = if (configuredMemory > 0) {
- if (preAllocateMemory) {
- LOG.info(s"Using $configuredMemory MB for managed memory.")
- } else {
- LOG.info(s"Limiting managed memory to $configuredMemory MB, " +
- s"memory will be allocated lazily.")
- }
- configuredMemory << 20 // megabytes to bytes
- }
- else {
- val fraction = configuration.getFloat(
- ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY,
- ConfigConstants.DEFAULT_MEMORY_MANAGER_MEMORY_FRACTION)
- checkConfigParameter(fraction > 0.0f && fraction < 1.0f, fraction,
- ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY,
- "MemoryManager fraction of the free memory must be between 0.0 and 1.0")
-
- if (memType == MemoryType.HEAP) {
- val relativeMemSize = (EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag() *
- fraction).toLong
-
- if (preAllocateMemory) {
- LOG.info(s"Using $fraction of the currently free heap space for managed " +
- s"heap memory (${relativeMemSize >> 20} MB).")
- } else {
- LOG.info(s"Limiting managed memory to $fraction of the currently free heap space " +
- s"(${relativeMemSize >> 20} MB), memory will be allocated lazily.")
- }
-
- relativeMemSize
- }
- else if (memType == MemoryType.OFF_HEAP) {
-
- // The maximum heap memory has been adjusted according to the fraction
- val maxMemory = EnvironmentInformation.getMaxJvmHeapMemory()
- val directMemorySize = (maxMemory / (1.0 - fraction) * fraction).toLong
-
- if (preAllocateMemory) {
- LOG.info(s"Using $fraction of the maximum memory size for " +
- s"managed off-heap memory (${directMemorySize >> 20} MB).")
- } else {
- LOG.info(s"Limiting managed memory to $fraction of the maximum memory size " +
- s"(${directMemorySize >> 20} MB), memory will be allocated lazily.")
- }
-
- directMemorySize
- }
- else {
- throw new RuntimeException("No supported memory type detected.")
- }
- }
-
- // now start the memory manager
- val memoryManager = try {
- new MemoryManager(
- memorySize,
- taskManagerConfig.numberOfSlots,
- netConfig.networkBufferSize,
- memType,
- preAllocateMemory)
- }
- catch {
- case e: OutOfMemoryError =>
- memType match {
- case MemoryType.HEAP =>
- throw new Exception(s"OutOfMemory error (${e.getMessage()})" +
- s" while allocating the TaskManager heap memory ($memorySize bytes).", e)
-
- case MemoryType.OFF_HEAP =>
- throw new Exception(s"OutOfMemory error (${e.getMessage()})" +
- s" while allocating the TaskManager off-heap memory ($memorySize bytes). " +
- s"Try increasing the maximum direct memory (-XX:MaxDirectMemorySize)", e)
-
- case _ => throw e
- }
- }
-
- // start the I/O manager last, it will create some temp directories.
- val ioManager: IOManager = new IOManagerAsync(taskManagerConfig.tmpDirPaths)
-
- val leaderRetrievalService = leaderRetrievalServiceOption match {
- case Some(lrs) => lrs
- case None => LeaderRetrievalUtils.createLeaderRetrievalService(configuration)
- }
-
- val metricsRegistry = new FlinkMetricRegistry(
- MetricRegistryConfiguration.fromConfiguration(configuration))
-
- (taskManagerConfig,
- taskManagerLocation,
- memoryManager,
- ioManager,
- network,
- leaderRetrievalService,
- metricsRegistry)
- }
-
-
// --------------------------------------------------------------------------
// Resolving the TaskManager actor
// --------------------------------------------------------------------------
@@ -2121,239 +1934,6 @@ object TaskManager {
// --------------------------------------------------------------------------
/**
- * Utility method to extract TaskManager config parameters from the configuration and to
- * sanity check them.
- *
- * @param configuration The configuration.
- * @param taskManagerHostname The host name under which the TaskManager communicates.
- * @param localTaskManagerCommunication True, to skip initializing the network stack.
- * Use only in cases where only one task manager runs.
- * @return A tuple (TaskManagerConfiguration, network configuration, inet socket address,
- * memory tyep).
- */
- @throws(classOf[IllegalArgumentException])
- def parseTaskManagerConfiguration(
- configuration: Configuration,
- taskManagerHostname: String,
- localTaskManagerCommunication: Boolean)
- : (TaskManagerConfiguration,
- NetworkEnvironmentConfiguration,
- InetSocketAddress,
- MemoryType) = {
-
- // ------- read values from the config and check them ---------
- // (a lot of them)
-
- // ----> hosts / ports for communication and data exchange
-
- val dataport = configuration.getInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_DATA_PORT)
-
- checkConfigParameter(dataport >= 0, dataport, ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
- "Leave config parameter empty or use 0 to let the system choose a port automatically.")
-
- val taskManagerAddress = InetAddress.getByName(taskManagerHostname)
- val taskManagerInetSocketAddress = new InetSocketAddress(taskManagerAddress, dataport)
-
- // ----> memory / network stack (shuffles/broadcasts), task slots, temp directories
-
- // we need this because many configs have been written with a "-1" entry
- val slots = configuration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1) match {
- case -1 => 1
- case x => x
- }
-
- checkConfigParameter(slots >= 1, slots, ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS,
- "Number of task slots must be at least one.")
-
- val numNetworkBuffers = configuration.getInteger(
- ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS)
-
- checkConfigParameter(numNetworkBuffers > 0, numNetworkBuffers,
- ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY)
-
- val pageSize: Int = configuration.getInteger(
- ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_SEGMENT_SIZE)
-
- // check page size of for minimum size
- checkConfigParameter(pageSize >= MemoryManager.MIN_PAGE_SIZE, pageSize,
- ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
- "Minimum memory segment size is " + MemoryManager.MIN_PAGE_SIZE)
-
- // check page size for power of two
- checkConfigParameter(MathUtils.isPowerOf2(pageSize), pageSize,
- ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
- "Memory segment size must be a power of 2.")
-
- // check whether we use heap or off-heap memory
- val memType: MemoryType =
- if (configuration.getBoolean(ConfigConstants.TASK_MANAGER_MEMORY_OFF_HEAP_KEY, false)) {
- MemoryType.OFF_HEAP
- } else {
- MemoryType.HEAP
- }
-
- // initialize the memory segment factory accordingly
- memType match {
- case MemoryType.HEAP =>
- if (!MemorySegmentFactory.initializeIfNotInitialized(HeapMemorySegment.FACTORY)) {
- throw new Exception("Memory type is set to heap memory, but memory segment " +
- "factory has been initialized for off-heap memory segments")
- }
-
- case MemoryType.OFF_HEAP =>
- if (!MemorySegmentFactory.initializeIfNotInitialized(HybridMemorySegment.FACTORY)) {
- throw new Exception("Memory type is set to off-heap memory, but memory segment " +
- "factory has been initialized for heap memory segments")
- }
- }
-
- val tmpDirs = configuration.getString(
- ConfigConstants.TASK_MANAGER_TMP_DIR_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH)
- .split(",|" + File.pathSeparator)
-
- val nettyConfig = if (localTaskManagerCommunication) {
- None
- } else {
- Some(
- new NettyConfig(
- taskManagerInetSocketAddress.getAddress(),
- taskManagerInetSocketAddress.getPort(),
- pageSize,
- slots,
- configuration)
- )
- }
-
- // Default spill I/O mode for intermediate results
- val syncOrAsync = configuration.getString(
- ConfigConstants.TASK_MANAGER_NETWORK_DEFAULT_IO_MODE,
- ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_DEFAULT_IO_MODE)
-
- val ioMode : IOMode = if (syncOrAsync == "async") IOMode.ASYNC else IOMode.SYNC
-
- val queryServerPort = configuration.getInteger(
- ConfigConstants.QUERYABLE_STATE_SERVER_PORT,
- ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_PORT)
-
- val queryServerNetworkThreads = configuration.getInteger(
- ConfigConstants.QUERYABLE_STATE_SERVER_NETWORK_THREADS,
- ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_NETWORK_THREADS)
-
- val queryServerQueryThreads = configuration.getInteger(
- ConfigConstants.QUERYABLE_STATE_SERVER_QUERY_THREADS,
- ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_QUERY_THREADS)
-
- val networkConfig = NetworkEnvironmentConfiguration(
- numNetworkBuffers,
- pageSize,
- memType,
- ioMode,
- queryServerPort,
- queryServerNetworkThreads,
- queryServerQueryThreads,
- nettyConfig.getOrElse(null))
-
- // ----> timeouts, library caching, profiling
-
- val timeout = try {
- AkkaUtils.getTimeout(configuration)
- } catch {
- case e: Exception => throw new IllegalArgumentException(
- s"Invalid format for '${ConfigConstants.AKKA_ASK_TIMEOUT}'. " +
- s"Use formats like '50 s' or '1 min' to specify the timeout.")
- }
- LOG.info("Messages between TaskManager and JobManager have a max timeout of " + timeout)
-
- val cleanupInterval = configuration.getLong(
- ConfigConstants.LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL,
- ConfigConstants.DEFAULT_LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL) * 1000
-
- val finiteRegistrationDuration = try {
- val maxRegistrationDuration = Duration(configuration.getString(
- ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION,
- ConfigConstants.DEFAULT_TASK_MANAGER_MAX_REGISTRATION_DURATION))
-
- if (maxRegistrationDuration.isFinite()) {
- Some(maxRegistrationDuration.asInstanceOf[FiniteDuration])
- } else {
- None
- }
- } catch {
- case e: NumberFormatException => throw new IllegalArgumentException(
- "Invalid format for parameter " + ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION,
- e)
- }
-
- val initialRegistrationPause = try {
- val pause = Duration(configuration.getString(
- ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE,
- ConfigConstants.DEFAULT_TASK_MANAGER_INITIAL_REGISTRATION_PAUSE
- ))
-
- if (pause.isFinite()) {
- pause.asInstanceOf[FiniteDuration]
- } else {
- throw new IllegalArgumentException(s"The initial registration pause must be finite: $pause")
- }
- } catch {
- case e: NumberFormatException => throw new IllegalArgumentException(
- "Invalid format for parameter " + ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE,
- e)
- }
-
- val maxRegistrationPause = try {
- val pause = Duration(configuration.getString(
- ConfigConstants.TASK_MANAGER_MAX_REGISTARTION_PAUSE,
- ConfigConstants.DEFAULT_TASK_MANAGER_MAX_REGISTRATION_PAUSE
- ))
-
- if (pause.isFinite()) {
- pause.asInstanceOf[FiniteDuration]
- } else {
- throw new IllegalArgumentException(s"The maximum registration pause must be finite: $pause")
- }
- } catch {
- case e: NumberFormatException => throw new IllegalArgumentException(
- "Invalid format for parameter " + ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE,
- e)
- }
-
- val refusedRegistrationPause = try {
- val pause = Duration(configuration.getString(
- ConfigConstants.TASK_MANAGER_REFUSED_REGISTRATION_PAUSE,
- ConfigConstants.DEFAULT_TASK_MANAGER_REFUSED_REGISTRATION_PAUSE
- ))
-
- if (pause.isFinite()) {
- pause.asInstanceOf[FiniteDuration]
- } else {
- throw new IllegalArgumentException(s"The refused registration pause must be finite: $pause")
- }
- } catch {
- case e: NumberFormatException => throw new IllegalArgumentException(
- "Invalid format for parameter " + ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE,
- e)
- }
-
- val taskManagerConfig = TaskManagerConfiguration(
- tmpDirs,
- cleanupInterval,
- timeout,
- finiteRegistrationDuration,
- slots,
- configuration,
- initialRegistrationPause,
- maxRegistrationPause,
- refusedRegistrationPause)
-
- (taskManagerConfig, networkConfig, taskManagerInetSocketAddress, memType)
- }
-
- /**
* Gets the protocol, hostname and port of the JobManager from the configuration. Also checks that
* the hostname is not null and the port non-negative.
*
@@ -2389,67 +1969,6 @@ object TaskManager {
// --------------------------------------------------------------------------
/**
- * Validates a condition for a config parameter and displays a standard exception, if the
- * the condition does not hold.
- *
- * @param condition The condition that must hold. If the condition is false, an
- * exception is thrown.
- * @param parameter The parameter value. Will be shown in the exception message.
- * @param name The name of the config parameter. Will be shown in the exception message.
- * @param errorMessage The optional custom error message to append to the exception message.
- * @throws IllegalConfigurationException Thrown if the condition is violated.
- */
- @throws(classOf[IllegalConfigurationException])
- private def checkConfigParameter(
- condition: Boolean,
- parameter: Any,
- name: String,
- errorMessage: String = "")
- : Unit = {
- if (!condition) {
- throw new IllegalConfigurationException(
- s"Invalid configuration value for '$name' : $parameter - $errorMessage")
- }
- }
-
- /**
- * Validates that all the directories denoted by the strings do actually exist, are proper
- * directories (not files), and are writable.
- *
- * @param tmpDirs The array of directory paths to check.
- * @throws Exception Thrown if any of the directories does not exist or is not writable
- * or is a file, rather than a directory.
- */
- @throws(classOf[IOException])
- private def checkTempDirs(tmpDirs: Array[String]): Unit = {
- tmpDirs.zipWithIndex.foreach {
- case (dir: String, _) =>
- val file = new File(dir)
-
- if (!file.exists) {
- throw new IOException(
- s"Temporary file directory ${file.getAbsolutePath} does not exist.")
- }
- if (!file.isDirectory) {
- throw new IOException(
- s"Temporary file directory ${file.getAbsolutePath} is not a directory.")
- }
- if (!file.canWrite) {
- throw new IOException(
- s"Temporary file directory ${file.getAbsolutePath} is not writable.")
- }
-
- if (LOG.isInfoEnabled) {
- val totalSpaceGb = file.getTotalSpace >> 30
- val usableSpaceGb = file.getUsableSpace >> 30
- val usablePercentage = usableSpaceGb.asInstanceOf[Double] / totalSpaceGb * 100
-
- val path = file.getAbsolutePath
-
- LOG.info(f"Temporary file directory '$path': total $totalSpaceGb GB, " +
- f"usable $usableSpaceGb GB ($usablePercentage%.2f%% usable)")
- }
- case (_, id) => throw new IllegalArgumentException(s"Temporary file directory #$id is null.")
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8e653397/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManagerConfiguration.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManagerConfiguration.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManagerConfiguration.scala
deleted file mode 100644
index aab3c5f..0000000
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManagerConfiguration.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.taskmanager
-
-import java.util.concurrent.TimeUnit
-
-import org.apache.flink.configuration.Configuration
-
-import scala.concurrent.duration.FiniteDuration
-
-case class TaskManagerConfiguration(
- tmpDirPaths: Array[String],
- cleanupInterval: Long,
- timeout: FiniteDuration,
- maxRegistrationDuration: Option[FiniteDuration],
- numberOfSlots: Int,
- configuration: Configuration,
- initialRegistrationPause: FiniteDuration,
- maxRegistrationPause: FiniteDuration,
- refusedRegistrationPause: FiniteDuration) {
-
- def this(
- tmpDirPaths: Array[String],
- cleanupInterval: Long,
- timeout: FiniteDuration,
- maxRegistrationDuration: Option[FiniteDuration],
- numberOfSlots: Int,
- configuration: Configuration) {
- this (
- tmpDirPaths,
- cleanupInterval,
- timeout,
- maxRegistrationDuration,
- numberOfSlots,
- configuration,
- FiniteDuration(500, TimeUnit.MILLISECONDS),
- FiniteDuration(30, TimeUnit.SECONDS),
- FiniteDuration(10, TimeUnit.SECONDS))
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/8e653397/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
index 627a25a..500d1bd 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
@@ -26,6 +26,7 @@ import akka.actor.Kill;
import akka.actor.Props;
import akka.testkit.JavaTestKit;
+import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.memory.MemoryType;
@@ -49,11 +50,11 @@ import org.apache.flink.runtime.messages.TaskManagerMessages;
import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
import org.apache.flink.runtime.query.KvStateRegistry;
+import org.apache.flink.runtime.taskexecutor.TaskManagerConfiguration;
import org.apache.flink.runtime.util.LeaderRetrievalUtils;
import org.junit.Test;
-import scala.Option;
import scala.concurrent.duration.FiniteDuration;
import java.net.InetAddress;
@@ -69,7 +70,7 @@ public class TaskManagerComponentsStartupShutdownTest {
public void testComponentsStartupShutdown() {
final String[] TMP_DIR = new String[] { ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH };
- final FiniteDuration timeout = new FiniteDuration(100, TimeUnit.SECONDS);
+ final Time timeout = Time.seconds(100);
final int BUFFER_SIZE = 32 * 1024;
Configuration config = new Configuration();
@@ -93,14 +94,19 @@ public class TaskManagerComponentsStartupShutdownTest {
LeaderRetrievalUtils.createLeaderRetrievalService(config, jobManager),
StandaloneResourceManager.class);
+ final int numberOfSlots = 1;
+
// create the components for the TaskManager manually
final TaskManagerConfiguration tmConfig = new TaskManagerConfiguration(
- TMP_DIR,
- 1000000,
- timeout,
- Option.<FiniteDuration>empty(),
- 1,
- config);
+ numberOfSlots,
+ TMP_DIR,
+ timeout,
+ null,
+ Time.milliseconds(500),
+ Time.seconds(30),
+ Time.seconds(10),
+ 1000000, // cleanup interval
+ config);
final NetworkEnvironmentConfiguration netConf = new NetworkEnvironmentConfiguration(
32, BUFFER_SIZE, MemoryType.HEAP, IOManager.IOMode.SYNC, 0, 0, 0,
@@ -125,8 +131,6 @@ public class TaskManagerComponentsStartupShutdownTest {
network.start();
- final int numberOfSlots = 1;
-
LeaderRetrievalService leaderRetrievalService = new StandaloneLeaderRetrievalService(jobManager.path().toString());
MetricRegistryConfiguration metricRegistryConfiguration = MetricRegistryConfiguration.fromConfiguration(config);
http://git-wip-us.apache.org/repos/asf/flink/blob/8e653397/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala
index 707401b..09dc5ed 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala
@@ -24,7 +24,8 @@ import org.apache.flink.runtime.io.network.NetworkEnvironment
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService
import org.apache.flink.runtime.memory.MemoryManager
import org.apache.flink.runtime.metrics.MetricRegistry
-import org.apache.flink.runtime.taskmanager.{TaskManagerLocation, TaskManager, TaskManagerConfiguration}
+import org.apache.flink.runtime.taskexecutor.TaskManagerConfiguration
+import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerLocation}
import scala.language.postfixOps
http://git-wip-us.apache.org/repos/asf/flink/blob/8e653397/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
index 73fb928..dba8834 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala
@@ -242,7 +242,6 @@ object TestingUtils {
)
}
-
def createTaskManager(
actorSystem: ActorSystem,
jobManagerURL: String,
http://git-wip-us.apache.org/repos/asf/flink/blob/8e653397/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala b/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
index 1010432..0f82faa 100644
--- a/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
+++ b/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
@@ -24,7 +24,8 @@ import org.apache.flink.runtime.io.network.NetworkEnvironment
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService
import org.apache.flink.runtime.memory.MemoryManager
import org.apache.flink.runtime.metrics.MetricRegistry
-import org.apache.flink.runtime.taskmanager.{TaskManagerConfiguration, TaskManagerLocation}
+import org.apache.flink.runtime.taskexecutor.TaskManagerConfiguration
+import org.apache.flink.runtime.taskmanager.TaskManagerLocation
import org.apache.flink.runtime.testingUtils.TestingTaskManagerLike
/** [[YarnTaskManager]] implementation which mixes in the [[TestingTaskManagerLike]] mixin.
http://git-wip-us.apache.org/repos/asf/flink/blob/8e653397/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
index 2ab9b20..be31085 100644
--- a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
+++ b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
@@ -23,8 +23,9 @@ import org.apache.flink.runtime.io.disk.iomanager.IOManager
import org.apache.flink.runtime.io.network.NetworkEnvironment
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService
import org.apache.flink.runtime.memory.MemoryManager
-import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerConfiguration, TaskManagerLocation}
+import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerLocation}
import org.apache.flink.runtime.metrics.MetricRegistry
+import org.apache.flink.runtime.taskexecutor.TaskManagerConfiguration
/** An extension of the TaskManager that listens for additional YARN related
* messages.
[04/50] [abbrv] flink git commit: [FLINK-4687] [rpc] Add getAddress
to RpcService
Posted by tr...@apache.org.
[FLINK-4687] [rpc] Add getAddress to RpcService
This closes #2551.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/56470067
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/56470067
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/56470067
Branch: refs/heads/flip-6
Commit: 5647006703b3cbbb12427da332d3d4f12312e8b3
Parents: 4f33185
Author: Till Rohrmann <tr...@apache.org>
Authored: Mon Sep 26 18:01:47 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:27 2016 +0100
----------------------------------------------------------------------
.../org/apache/flink/runtime/rpc/RpcService.java | 8 ++++++++
.../flink/runtime/rpc/akka/AkkaRpcService.java | 16 ++++++++++++++++
.../apache/flink/runtime/rpc/TestingRpcService.java | 5 +++--
.../flink/runtime/rpc/TestingSerialRpcService.java | 6 ++++++
.../flink/runtime/rpc/akka/AkkaRpcActorTest.java | 8 ++++----
.../flink/runtime/rpc/akka/AkkaRpcServiceTest.java | 5 +++++
6 files changed, 42 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/56470067/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java
index 437e08b..96844ed 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java
@@ -33,6 +33,14 @@ import java.util.concurrent.TimeUnit;
public interface RpcService {
/**
+ * Return the address under which the rpc service can be reached. If the rpc service cannot be
+ * contacted remotely, then it will return an empty string.
+ *
+ * @return Address of the rpc service or empty string if local rpc service
+ */
+ String getAddress();
+
+ /**
* Connect to a remote rpc server under the provided address. Returns a rpc gateway which can
* be used to communicate with the rpc server. If the connection failed, then the returned
* future is failed with a {@link RpcConnectionException}.
http://git-wip-us.apache.org/repos/asf/flink/blob/56470067/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
index cee19c4..6825557 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
@@ -22,6 +22,7 @@ import akka.actor.ActorIdentity;
import akka.actor.ActorRef;
import akka.actor.ActorSelection;
import akka.actor.ActorSystem;
+import akka.actor.Address;
import akka.actor.Identify;
import akka.actor.PoisonPill;
import akka.actor.Props;
@@ -75,6 +76,8 @@ public class AkkaRpcService implements RpcService {
private final Set<ActorRef> actors = new HashSet<>(4);
private final long maximumFramesize;
+ private final String address;
+
private volatile boolean stopped;
public AkkaRpcService(final ActorSystem actorSystem, final Time timeout) {
@@ -87,6 +90,19 @@ public class AkkaRpcService implements RpcService {
// only local communication
maximumFramesize = Long.MAX_VALUE;
}
+
+ Address actorSystemAddress = AkkaUtils.getAddress(actorSystem);
+
+ if (actorSystemAddress.host().isDefined()) {
+ address = actorSystemAddress.host().get();
+ } else {
+ address = "";
+ }
+ }
+
+ @Override
+ public String getAddress() {
+ return address;
}
// this method does not mutate state and is thus thread-safe
http://git-wip-us.apache.org/repos/asf/flink/blob/56470067/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingRpcService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingRpcService.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingRpcService.java
index f164056..47c9e24 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingRpcService.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingRpcService.java
@@ -25,6 +25,7 @@ import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
+import java.net.UnknownHostException;
import java.util.concurrent.ConcurrentHashMap;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -57,14 +58,14 @@ public class TestingRpcService extends AkkaRpcService {
/**
* Creates a new {@code TestingRpcService}.
*/
- public TestingRpcService() {
+ public TestingRpcService() throws UnknownHostException {
this(new Configuration());
}
/**
* Creates a new {@code TestingRpcService}, using the given configuration.
*/
- public TestingRpcService(Configuration configuration) {
+ public TestingRpcService(Configuration configuration) throws UnknownHostException {
super(AkkaUtils.createLocalActorSystem(configuration), Time.seconds(10));
this.registeredConnections = new ConcurrentHashMap<>();
http://git-wip-us.apache.org/repos/asf/flink/blob/56470067/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java
index c58ea20..5b8e6e6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java
@@ -30,6 +30,7 @@ import java.lang.annotation.Annotation;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
+import java.net.InetAddress;
import java.util.BitSet;
import java.util.UUID;
import java.util.concurrent.Callable;
@@ -121,6 +122,11 @@ public class TestingSerialRpcService implements RpcService {
}
@Override
+ public String getAddress() {
+ return "";
+ }
+
+ @Override
public <C extends RpcGateway> Future<C> connect(String address, Class<C> clazz) {
RpcGateway gateway = registeredConnections.get(address);
http://git-wip-us.apache.org/repos/asf/flink/blob/56470067/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
index 1e8c9a6..5d76024 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
@@ -133,7 +133,7 @@ public class AkkaRpcActorTest extends TestLogger {
Future<WrongRpcGateway> futureGateway = akkaRpcService.connect(rpcEndpoint.getAddress(), WrongRpcGateway.class);
- WrongRpcGateway gateway = Await.result(futureGateway, timeout.duration());
+ WrongRpcGateway gateway = futureGateway.get(timeout.getSize(), timeout.getUnit());
// since it is a tell operation we won't receive a RpcConnectionException, it's only logged
gateway.tell("foobar");
@@ -141,10 +141,10 @@ public class AkkaRpcActorTest extends TestLogger {
Future<Boolean> result = gateway.barfoo();
try {
- Await.result(result, timeout.duration());
+ result.get(timeout.getSize(), timeout.getUnit());
fail("We expected a RpcConnectionException.");
- } catch (RpcConnectionException rpcConnectionException) {
- // we expect this exception here
+ } catch (ExecutionException executionException) {
+ assertTrue(executionException.getCause() instanceof RpcConnectionException);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/56470067/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceTest.java
index 5550cb5..3388011 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceTest.java
@@ -115,4 +115,9 @@ public class AkkaRpcServiceTest extends TestLogger {
assertEquals(expected, actual);
assertTrue(latch.isTriggered());
}
+
+ @Test
+ public void testGetAddress() {
+ assertEquals(AkkaUtils.getAddress(actorSystem).host().get(), akkaRpcService.getAddress());
+ }
}
[12/50] [abbrv] flink git commit: [FLINK-4505] [cluster mngt]
Separate TaskManager service configuration from TaskManagerConfiguration;
Implement TaskManagerRunner
Posted by tr...@apache.org.
[FLINK-4505] [cluster mngt] Separate TaskManager service configuration from TaskManagerConfiguration; Implement TaskManagerRunner
Refactors the startup logic so that is easier to reuse.
This closes #2461.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5ca94fb9
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5ca94fb9
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5ca94fb9
Branch: refs/heads/flip-6
Commit: 5ca94fb9a499ff5ffec333270e2eb60acd4e008a
Parents: 0a32c6c
Author: Till Rohrmann <tr...@apache.org>
Authored: Wed Sep 21 12:33:15 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:28 2016 +0100
----------------------------------------------------------------------
.../HighAvailabilityServicesUtils.java | 41 +
.../flink/runtime/rpc/RpcServiceUtils.java | 73 ++
.../flink/runtime/rpc/akka/AkkaRpcService.java | 2 +
.../runtime/taskexecutor/TaskExecutor.java | 51 +-
.../taskexecutor/TaskExecutorConfiguration.java | 142 ----
.../taskexecutor/TaskManagerConfiguration.java | 205 +++++
.../runtime/taskexecutor/TaskManagerRunner.java | 172 +++++
.../taskexecutor/TaskManagerServices.java | 320 ++++++++
.../TaskManagerServicesConfiguration.java | 325 ++++++++
.../runtime/taskmanager/TaskManagerRunner.java | 749 -------------------
.../runtime/util/LeaderRetrievalUtils.java | 7 +
.../apache/flink/runtime/akka/AkkaUtils.scala | 4 +
.../NetworkEnvironmentConfiguration.scala | 2 +-
.../flink/runtime/taskmanager/TaskManager.scala | 6 +-
.../io/network/NetworkEnvironmentTest.java | 4 +-
.../runtime/rpc/TestingSerialRpcService.java | 1 -
.../runtime/taskexecutor/TaskExecutorTest.java | 29 +-
...askManagerComponentsStartupShutdownTest.java | 3 +-
.../TaskManagerConfigurationTest.java | 1 -
19 files changed, 1195 insertions(+), 942 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java
new file mode 100644
index 0000000..f3da847
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.highavailability;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
+import org.apache.flink.runtime.util.LeaderRetrievalUtils;
+
+public class HighAvailabilityServicesUtils {
+
+ public static HighAvailabilityServices createHighAvailabilityServices(Configuration configuration) throws Exception {
+ HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(configuration);
+
+ switch(highAvailabilityMode) {
+ case NONE:
+ final String resourceManagerAddress = null;
+ return new NonHaServices(resourceManagerAddress);
+ case ZOOKEEPER:
+ throw new UnsupportedOperationException("ZooKeeper high availability services " +
+ "have not been implemented yet.");
+ default:
+ throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcServiceUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcServiceUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcServiceUtils.java
new file mode 100644
index 0000000..d40e336
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcServiceUtils.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rpc;
+
+import akka.actor.ActorSystem;
+import com.typesafe.config.Config;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
+import org.apache.flink.util.NetUtils;
+import org.jboss.netty.channel.ChannelException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class RpcServiceUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(RpcServiceUtils.class);
+
+ /**
+ * Utility method to create RPC service from configuration and hostname, port.
+ *
+ * @param hostname The hostname/address that describes the TaskManager's data location.
+ * @param port If true, the TaskManager will not initiate the TCP network stack.
+ * @param configuration The configuration for the TaskManager.
+ * @return The rpc service which is used to start and connect to the TaskManager RpcEndpoint .
+ * @throws IOException Thrown, if the actor system can not bind to the address
+ * @throws Exception Thrown is some other error occurs while creating akka actor system
+ */
+ public static RpcService createRpcService(String hostname, int port, Configuration configuration) throws Exception {
+ LOG.info("Starting AkkaRpcService at {}.", NetUtils.hostAndPortToUrlString(hostname, port));
+
+ final ActorSystem actorSystem;
+
+ try {
+ Config akkaConfig = AkkaUtils.getAkkaConfig(configuration, hostname, port);
+
+ LOG.debug("Using akka configuration \n {}.", akkaConfig);
+
+ actorSystem = AkkaUtils.createActorSystem(akkaConfig);
+ } catch (Throwable t) {
+ if (t instanceof ChannelException) {
+ Throwable cause = t.getCause();
+ if (cause != null && t.getCause() instanceof java.net.BindException) {
+ String address = NetUtils.hostAndPortToUrlString(hostname, port);
+ throw new IOException("Unable to bind AkkaRpcService actor system to address " +
+ address + " - " + cause.getMessage(), t);
+ }
+ }
+ throw new Exception("Could not create TaskManager actor system", t);
+ }
+
+ final Time timeout = Time.milliseconds(AkkaUtils.getTimeout(configuration).toMillis());
+ return new AkkaRpcService(actorSystem, timeout);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
index 6825557..fb7896a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
@@ -93,6 +93,8 @@ public class AkkaRpcService implements RpcService {
Address actorSystemAddress = AkkaUtils.getAddress(actorSystem);
+
+
if (actorSystemAddress.host().isDefined()) {
address = actorSystemAddress.host().get();
} else {
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index 8ce2780..7df0a91 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -18,16 +18,14 @@
package org.apache.flink.runtime.taskexecutor;
-import org.apache.flink.api.common.time.Time;
-import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
-import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.resourcemanager.SlotRequestRegistered;
import org.apache.flink.runtime.resourcemanager.SlotRequestReply;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
-import org.jboss.netty.channel.ChannelException;
import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.network.NetworkEnvironment;
@@ -39,7 +37,7 @@ import org.apache.flink.runtime.rpc.RpcService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.net.BindException;
+
import java.util.UUID;
import static org.apache.flink.util.Preconditions.checkArgument;
@@ -60,7 +58,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
private final HighAvailabilityServices haServices;
/** The task manager configuration */
- private final TaskExecutorConfiguration taskExecutorConfig;
+ private final TaskManagerConfiguration taskManagerConfiguration;
/** The I/O manager component in the task manager */
private final IOManager ioManager;
@@ -71,9 +69,15 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
/** The network component in the task manager */
private final NetworkEnvironment networkEnvironment;
+ /** The metric registry in the task manager */
+ private final MetricRegistry metricRegistry;
+
/** The number of slots in the task manager, should be 1 for YARN */
private final int numberOfSlots;
+ /** The fatal error handler to use in case of a fatal error */
+ private final FatalErrorHandler fatalErrorHandler;
+
// --------- resource manager --------
private TaskExecutorToResourceManagerConnection resourceManagerConnection;
@@ -81,26 +85,30 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
// ------------------------------------------------------------------------
public TaskExecutor(
- TaskExecutorConfiguration taskExecutorConfig,
+ TaskManagerConfiguration taskManagerConfiguration,
TaskManagerLocation taskManagerLocation,
RpcService rpcService,
MemoryManager memoryManager,
IOManager ioManager,
NetworkEnvironment networkEnvironment,
- HighAvailabilityServices haServices) {
+ HighAvailabilityServices haServices,
+ MetricRegistry metricRegistry,
+ FatalErrorHandler fatalErrorHandler) {
super(rpcService);
- checkArgument(taskExecutorConfig.getNumberOfSlots() > 0, "The number of slots has to be larger than 0.");
+ checkArgument(taskManagerConfiguration.getNumberSlots() > 0, "The number of slots has to be larger than 0.");
- this.taskExecutorConfig = checkNotNull(taskExecutorConfig);
+ this.taskManagerConfiguration = checkNotNull(taskManagerConfiguration);
this.taskManagerLocation = checkNotNull(taskManagerLocation);
this.memoryManager = checkNotNull(memoryManager);
this.ioManager = checkNotNull(ioManager);
this.networkEnvironment = checkNotNull(networkEnvironment);
this.haServices = checkNotNull(haServices);
+ this.metricRegistry = checkNotNull(metricRegistry);
+ this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
- this.numberOfSlots = taskExecutorConfig.getNumberOfSlots();
+ this.numberOfSlots = taskManagerConfiguration.getNumberSlots();
}
// ------------------------------------------------------------------------
@@ -158,6 +166,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
+ /**
* Requests a slot from the TaskManager
*
* @param allocationID id for the request
@@ -169,22 +178,6 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
return new SlotRequestRegistered(allocationID);
}
- /**
- public LeaderRetrievalService getJobMasterLeaderRetriever(JobID jobID) throws Exception {
- return null;
- }
-
- @Override
- return null;
- }
-
- @Override
- public CheckpointRecoveryFactory getCheckpointRecoveryFactory() throws Exception {
- return null;
- }
-
- @Override
- public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
// ------------------------------------------------------------------------
// Properties
// ------------------------------------------------------------------------
@@ -222,7 +215,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
void onFatalError(Throwable t) {
// to be determined, probably delegate to a fatal error handler that
// would either log (mini cluster) ot kill the process (yarn, mesos, ...)
- log.error("FATAL ERROR", t);
+ fatalErrorHandler.onFatalError(t);
}
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorConfiguration.java
deleted file mode 100644
index c97c893..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorConfiguration.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.taskexecutor;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration;
-
-import scala.concurrent.duration.FiniteDuration;
-
-import java.io.Serializable;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/**
- * {@link TaskExecutor} Configuration
- */
-public class TaskExecutorConfiguration implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private final String[] tmpDirPaths;
-
- private final long cleanupInterval;
-
- private final int numberOfSlots;
-
- private final Configuration configuration;
-
- private final FiniteDuration timeout;
- private final FiniteDuration maxRegistrationDuration;
- private final FiniteDuration initialRegistrationPause;
- private final FiniteDuration maxRegistrationPause;
- private final FiniteDuration refusedRegistrationPause;
-
- private final NetworkEnvironmentConfiguration networkConfig;
-
- public TaskExecutorConfiguration(
- String[] tmpDirPaths,
- long cleanupInterval,
- NetworkEnvironmentConfiguration networkConfig,
- FiniteDuration timeout,
- FiniteDuration maxRegistrationDuration,
- int numberOfSlots,
- Configuration configuration) {
-
- this (tmpDirPaths,
- cleanupInterval,
- networkConfig,
- timeout,
- maxRegistrationDuration,
- numberOfSlots,
- configuration,
- new FiniteDuration(500, TimeUnit.MILLISECONDS),
- new FiniteDuration(30, TimeUnit.SECONDS),
- new FiniteDuration(10, TimeUnit.SECONDS));
- }
-
- public TaskExecutorConfiguration(
- String[] tmpDirPaths,
- long cleanupInterval,
- NetworkEnvironmentConfiguration networkConfig,
- FiniteDuration timeout,
- FiniteDuration maxRegistrationDuration,
- int numberOfSlots,
- Configuration configuration,
- FiniteDuration initialRegistrationPause,
- FiniteDuration maxRegistrationPause,
- FiniteDuration refusedRegistrationPause) {
-
- this.tmpDirPaths = checkNotNull(tmpDirPaths);
- this.cleanupInterval = checkNotNull(cleanupInterval);
- this.networkConfig = checkNotNull(networkConfig);
- this.timeout = checkNotNull(timeout);
- this.maxRegistrationDuration = maxRegistrationDuration;
- this.numberOfSlots = checkNotNull(numberOfSlots);
- this.configuration = checkNotNull(configuration);
- this.initialRegistrationPause = checkNotNull(initialRegistrationPause);
- this.maxRegistrationPause = checkNotNull(maxRegistrationPause);
- this.refusedRegistrationPause = checkNotNull(refusedRegistrationPause);
- }
-
- // --------------------------------------------------------------------------------------------
- // Properties
- // --------------------------------------------------------------------------------------------
-
- public String[] getTmpDirPaths() {
- return tmpDirPaths;
- }
-
- public long getCleanupInterval() {
- return cleanupInterval;
- }
-
- public NetworkEnvironmentConfiguration getNetworkConfig() { return networkConfig; }
-
- public FiniteDuration getTimeout() {
- return timeout;
- }
-
- public FiniteDuration getMaxRegistrationDuration() {
- return maxRegistrationDuration;
- }
-
- public int getNumberOfSlots() {
- return numberOfSlots;
- }
-
- public Configuration getConfiguration() {
- return configuration;
- }
-
- public FiniteDuration getInitialRegistrationPause() {
- return initialRegistrationPause;
- }
-
- public FiniteDuration getMaxRegistrationPause() {
- return maxRegistrationPause;
- }
-
- public FiniteDuration getRefusedRegistrationPause() {
- return refusedRegistrationPause;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
new file mode 100644
index 0000000..32eb8c1
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor;
+
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.concurrent.duration.Duration;
+
+import java.io.File;
+
+/**
+ * Configuration object for {@link TaskExecutor}.
+ */
+public class TaskManagerConfiguration {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TaskManagerConfiguration.class);
+
+ private final int numberSlots;
+
+ private final String[] tmpDirPaths;
+
+ private final Time timeout;
+ private final Time maxRegistrationDuration;
+ private final Time initialRegistrationPause;
+ private final Time maxRegistrationPause;
+ private final Time refusedRegistrationPause;
+
+ private final long cleanupInterval;
+
+ public TaskManagerConfiguration(
+ int numberSlots,
+ String[] tmpDirPaths,
+ Time timeout,
+ Time maxRegistrationDuration,
+ Time initialRegistrationPause,
+ Time maxRegistrationPause,
+ Time refusedRegistrationPause,
+ long cleanupInterval) {
+
+ this.numberSlots = numberSlots;
+ this.tmpDirPaths = Preconditions.checkNotNull(tmpDirPaths);
+ this.timeout = Preconditions.checkNotNull(timeout);
+ this.maxRegistrationDuration = Preconditions.checkNotNull(maxRegistrationDuration);
+ this.initialRegistrationPause = Preconditions.checkNotNull(initialRegistrationPause);
+ this.maxRegistrationPause = Preconditions.checkNotNull(maxRegistrationPause);
+ this.refusedRegistrationPause = Preconditions.checkNotNull(refusedRegistrationPause);
+ this.cleanupInterval = Preconditions.checkNotNull(cleanupInterval);
+ }
+
+ public int getNumberSlots() {
+ return numberSlots;
+ }
+
+ public String[] getTmpDirPaths() {
+ return tmpDirPaths;
+ }
+
+ public Time getTimeout() {
+ return timeout;
+ }
+
+ public Time getMaxRegistrationDuration() {
+ return maxRegistrationDuration;
+ }
+
+ public Time getInitialRegistrationPause() {
+ return initialRegistrationPause;
+ }
+
+ public Time getMaxRegistrationPause() {
+ return maxRegistrationPause;
+ }
+
+ public Time getRefusedRegistrationPause() {
+ return refusedRegistrationPause;
+ }
+
+ public long getCleanupInterval() {
+ return cleanupInterval;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Static factory methods
+ // --------------------------------------------------------------------------------------------
+
+ public static TaskManagerConfiguration fromConfiguration(Configuration configuration) {
+ int numberSlots = configuration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
+
+ if (numberSlots == -1) {
+ numberSlots = 1;
+ }
+
+ final String[] tmpDirPaths = configuration.getString(
+ ConfigConstants.TASK_MANAGER_TMP_DIR_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH).split(",|" + File.pathSeparator);
+
+ final Time timeout;
+
+ try {
+ timeout = Time.milliseconds(AkkaUtils.getTimeout(configuration).toMillis());
+ } catch (Exception e) {
+ throw new IllegalArgumentException(
+ "Invalid format for '" + ConfigConstants.AKKA_ASK_TIMEOUT +
+ "'.Use formats like '50 s' or '1 min' to specify the timeout.");
+ }
+
+ LOG.info("Messages have a max timeout of " + timeout);
+
+ final long cleanupInterval = configuration.getLong(
+ ConfigConstants.LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL,
+ ConfigConstants.DEFAULT_LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL) * 1000;
+
+ final Time finiteRegistrationDuration;
+
+ try {
+ Duration maxRegistrationDuration = Duration.create(configuration.getString(
+ ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION,
+ ConfigConstants.DEFAULT_TASK_MANAGER_MAX_REGISTRATION_DURATION));
+ if (maxRegistrationDuration.isFinite()) {
+ finiteRegistrationDuration = Time.seconds(maxRegistrationDuration.toSeconds());
+ } else {
+ finiteRegistrationDuration = null;
+ }
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Invalid format for parameter " +
+ ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION, e);
+ }
+
+ final Time initialRegistrationPause;
+ try {
+ Duration pause = Duration.create(configuration.getString(
+ ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE,
+ ConfigConstants.DEFAULT_TASK_MANAGER_INITIAL_REGISTRATION_PAUSE));
+ if (pause.isFinite()) {
+ initialRegistrationPause = Time.seconds(pause.toSeconds());
+ } else {
+ throw new IllegalArgumentException("The initial registration pause must be finite: " + pause);
+ }
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Invalid format for parameter " +
+ ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
+ }
+
+ final Time maxRegistrationPause;
+ try {
+ Duration pause = Duration.create(configuration.getString(
+ ConfigConstants.TASK_MANAGER_MAX_REGISTARTION_PAUSE,
+ ConfigConstants.DEFAULT_TASK_MANAGER_MAX_REGISTRATION_PAUSE));
+ if (pause.isFinite()) {
+ maxRegistrationPause = Time.seconds(pause.toSeconds());
+ } else {
+ throw new IllegalArgumentException("The maximum registration pause must be finite: " + pause);
+ }
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Invalid format for parameter " +
+ ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
+ }
+
+ final Time refusedRegistrationPause;
+ try {
+ Duration pause = Duration.create(configuration.getString(
+ ConfigConstants.TASK_MANAGER_REFUSED_REGISTRATION_PAUSE,
+ ConfigConstants.DEFAULT_TASK_MANAGER_REFUSED_REGISTRATION_PAUSE));
+ if (pause.isFinite()) {
+ refusedRegistrationPause = Time.seconds(pause.toSeconds());
+ } else {
+ throw new IllegalArgumentException("The refused registration pause must be finite: " + pause);
+ }
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Invalid format for parameter " +
+ ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
+ }
+
+ return new TaskManagerConfiguration(
+ numberSlots,
+ tmpDirPaths,
+ timeout,
+ finiteRegistrationDuration,
+ initialRegistrationPause,
+ maxRegistrationPause,
+ refusedRegistrationPause,
+ cleanupInterval);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
new file mode 100644
index 0000000..8ac0ddd
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor;
+
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
+import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.runtime.rpc.RpcServiceUtils;
+import org.apache.flink.runtime.util.LeaderRetrievalUtils;
+
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.util.concurrent.Executor;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * This class is the executable entry point for the task manager in yarn or standalone mode.
+ * It constructs the related components (network, I/O manager, memory manager, RPC service, HA service)
+ * and starts them.
+ */
+public class TaskManagerRunner implements FatalErrorHandler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TaskManagerRunner.class);
+
+ private final Object lock = new Object();
+
+ private final Configuration configuration;
+
+ private final ResourceID resourceID;
+
+ private final RpcService rpcService;
+
+ private final HighAvailabilityServices highAvailabilityServices;
+
+ /** Executor used to run future callbacks */
+ private final Executor executor;
+
+ private final TaskExecutor taskManager;
+
+ public TaskManagerRunner(
+ Configuration configuration,
+ ResourceID resourceID,
+ RpcService rpcService,
+ HighAvailabilityServices highAvailabilityServices,
+ Executor executor) throws Exception {
+
+ this.configuration = Preconditions.checkNotNull(configuration);
+ this.resourceID = Preconditions.checkNotNull(resourceID);
+ this.rpcService = Preconditions.checkNotNull(rpcService);
+ this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices);
+ this.executor = rpcService.getExecutor();
+
+ InetAddress remoteAddress = InetAddress.getByName(rpcService.getAddress());
+
+ TaskManagerServicesConfiguration taskManagerServicesConfiguration = TaskManagerServicesConfiguration.fromConfiguration(
+ configuration,
+ remoteAddress,
+ false);
+
+ TaskManagerServices taskManagerServices = TaskManagerServices.fromConfiguration(taskManagerServicesConfiguration, resourceID);
+
+ TaskManagerConfiguration taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration);
+
+ this.taskManager = new TaskExecutor(
+ taskManagerConfiguration,
+ taskManagerServices.getTaskManagerLocation(),
+ rpcService,
+ taskManagerServices.getMemoryManager(),
+ taskManagerServices.getIOManager(),
+ taskManagerServices.getNetworkEnvironment(),
+ highAvailabilityServices,
+ taskManagerServices.getMetricRegistry(),
+ this);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Lifecycle management
+ // --------------------------------------------------------------------------------------------
+
+ public void start() {
+ taskManager.start();
+ }
+
+ public void shutDown(Throwable cause) {
+ shutDownInternally();
+ }
+
+ protected void shutDownInternally() {
+ synchronized(lock) {
+ taskManager.shutDown();
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // FatalErrorHandler methods
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void onFatalError(Throwable exception) {
+ LOG.error("Fatal error occurred while executing the TaskManager. Shutting it down...", exception);
+ shutDown(exception);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Static utilities
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Create a RPC service for the task manager.
+ *
+ * @param configuration The configuration for the TaskManager.
+ * @param haServices to use for the task manager hostname retrieval
+ */
+ public static RpcService createRpcService(
+ final Configuration configuration,
+ final HighAvailabilityServices haServices) throws Exception {
+
+ checkNotNull(configuration);
+ checkNotNull(haServices);
+
+ String taskManagerHostname = configuration.getString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, null);
+
+ if (taskManagerHostname != null) {
+ LOG.info("Using configured hostname/address for TaskManager: {}.", taskManagerHostname);
+ } else {
+ Time lookupTimeout = Time.milliseconds(AkkaUtils.getLookupTimeout(configuration).toMillis());
+
+ InetAddress taskManagerAddress = LeaderRetrievalUtils.findConnectingAddress(
+ haServices.getResourceManagerLeaderRetriever(),
+ lookupTimeout);
+
+ taskManagerHostname = taskManagerAddress.getHostName();
+
+ LOG.info("TaskManager will use hostname/address '{}' ({}) for communication.",
+ taskManagerHostname, taskManagerAddress.getHostAddress());
+ }
+
+ final int rpcPort = configuration.getInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, 0);
+
+ Preconditions.checkState(rpcPort < 0 || rpcPort >65535, "Invalid value for " +
+ "'%s' (port for the TaskManager actor system) : %d - Leave config parameter empty or " +
+ "use 0 to let the system choose port automatically.",
+ ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, rpcPort);
+
+ return RpcServiceUtils.createRpcService(taskManagerHostname, rpcPort, configuration);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
new file mode 100644
index 0000000..ff7f7d5
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor;
+
+import org.apache.flink.core.memory.MemoryType;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.io.disk.iomanager.IOManager;
+import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync;
+import org.apache.flink.runtime.io.network.ConnectionManager;
+import org.apache.flink.runtime.io.network.LocalConnectionManager;
+import org.apache.flink.runtime.io.network.NetworkEnvironment;
+import org.apache.flink.runtime.io.network.TaskEventDispatcher;
+import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
+import org.apache.flink.runtime.io.network.netty.NettyConfig;
+import org.apache.flink.runtime.io.network.netty.NettyConnectionManager;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionManager;
+import org.apache.flink.runtime.memory.MemoryManager;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.query.KvStateRegistry;
+import org.apache.flink.runtime.query.netty.DisabledKvStateRequestStats;
+import org.apache.flink.runtime.query.netty.KvStateServer;
+import org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration;
+import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
+import org.apache.flink.runtime.util.EnvironmentInformation;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Container for {@link TaskExecutor} services such as the {@link MemoryManager}, {@link IOManager},
+ * {@link NetworkEnvironment} and the {@link MetricRegistry}.
+ */
+public class TaskManagerServices {
+ private static final Logger LOG = LoggerFactory.getLogger(TaskManagerServices.class);
+
+ /** TaskManager services */
+ private final TaskManagerLocation taskManagerLocation;
+ private final MemoryManager memoryManager;
+ private final IOManager ioManager;
+ private final NetworkEnvironment networkEnvironment;
+ private final MetricRegistry metricRegistry;
+
+ private TaskManagerServices(
+ TaskManagerLocation taskManagerLocation,
+ MemoryManager memoryManager,
+ IOManager ioManager,
+ NetworkEnvironment networkEnvironment,
+ MetricRegistry metricRegistry) {
+
+ this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
+ this.memoryManager = Preconditions.checkNotNull(memoryManager);
+ this.ioManager = Preconditions.checkNotNull(ioManager);
+ this.networkEnvironment = Preconditions.checkNotNull(networkEnvironment);
+ this.metricRegistry = Preconditions.checkNotNull(metricRegistry);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Getter/Setter
+ // --------------------------------------------------------------------------------------------
+
+ public MemoryManager getMemoryManager() {
+ return memoryManager;
+ }
+
+ public IOManager getIOManager() {
+ return ioManager;
+ }
+
+ public NetworkEnvironment getNetworkEnvironment() {
+ return networkEnvironment;
+ }
+
+ public TaskManagerLocation getTaskManagerLocation() {
+ return taskManagerLocation;
+ }
+
+ public MetricRegistry getMetricRegistry() {
+ return metricRegistry;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Static factory methods for task manager services
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Creates and returns the task manager services.
+ *
+ * @param resourceID resource ID of the task manager
+ * @param taskManagerServicesConfiguration task manager configuration
+ * @return task manager components
+ * @throws Exception
+ */
+ public static TaskManagerServices fromConfiguration(
+ TaskManagerServicesConfiguration taskManagerServicesConfiguration,
+ ResourceID resourceID) throws Exception {
+
+ final NetworkEnvironment network = createNetworkEnvironment(taskManagerServicesConfiguration);
+
+ network.start();
+
+ final TaskManagerLocation taskManagerLocation = new TaskManagerLocation(
+ resourceID,
+ taskManagerServicesConfiguration.getTaskManagerAddress(),
+ network.getConnectionManager().getDataPort());
+
+ // this call has to happen strictly after the network stack has been initialized
+ final MemoryManager memoryManager = createMemoryManager(taskManagerServicesConfiguration);
+
+ // start the I/O manager, it will create some temp directories.
+ final IOManager ioManager = new IOManagerAsync(taskManagerServicesConfiguration.getTmpDirPaths());
+
+ MetricRegistry metricsRegistry = new MetricRegistry(taskManagerServicesConfiguration.getMetricRegistryConfiguration());
+
+ return new TaskManagerServices(taskManagerLocation, memoryManager, ioManager, network, metricsRegistry);
+ }
+
+ /**
+ * Creates a {@link MemoryManager} from the given {@link TaskManagerServicesConfiguration}.
+ *
+ * @param taskManagerServicesConfiguration to create the memory manager from
+ * @return Memory manager
+ * @throws Exception
+ */
+ private static MemoryManager createMemoryManager(TaskManagerServicesConfiguration taskManagerServicesConfiguration) throws Exception {
+ // computing the amount of memory to use depends on how much memory is available
+ // it strictly needs to happen AFTER the network stack has been initialized
+
+ MemoryType memType = taskManagerServicesConfiguration.getNetworkConfig().memoryType();
+
+ // check if a value has been configured
+ long configuredMemory = taskManagerServicesConfiguration.getConfiguredMemory();
+
+ final long memorySize;
+
+ boolean preAllocateMemory = taskManagerServicesConfiguration.isPreAllocateMemory();
+
+ if (configuredMemory > 0) {
+ if (preAllocateMemory) {
+ LOG.info("Using {} MB for managed memory." , configuredMemory);
+ } else {
+ LOG.info("Limiting managed memory to {} MB, memory will be allocated lazily." , configuredMemory);
+ }
+ memorySize = configuredMemory << 20; // megabytes to bytes
+ } else {
+ float memoryFraction = taskManagerServicesConfiguration.getMemoryFraction();
+
+ if (memType == MemoryType.HEAP) {
+ long relativeMemSize = (long) (EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag() * memoryFraction);
+ if (preAllocateMemory) {
+ LOG.info("Using {} of the currently free heap space for managed heap memory ({} MB)." ,
+ memoryFraction , relativeMemSize >> 20);
+ } else {
+ LOG.info("Limiting managed memory to {} of the currently free heap space ({} MB), " +
+ "memory will be allocated lazily." , memoryFraction , relativeMemSize >> 20);
+ }
+ memorySize = relativeMemSize;
+ } else if (memType == MemoryType.OFF_HEAP) {
+ // The maximum heap memory has been adjusted according to the fraction
+ long maxMemory = EnvironmentInformation.getMaxJvmHeapMemory();
+ long directMemorySize = (long) (maxMemory / (1.0 - memoryFraction) * memoryFraction);
+ if (preAllocateMemory) {
+ LOG.info("Using {} of the maximum memory size for managed off-heap memory ({} MB)." ,
+ memoryFraction, directMemorySize >> 20);
+ } else {
+ LOG.info("Limiting managed memory to {} of the maximum memory size ({} MB)," +
+ " memory will be allocated lazily.", memoryFraction, directMemorySize >> 20);
+ }
+ memorySize = directMemorySize;
+ } else {
+ throw new RuntimeException("No supported memory type detected.");
+ }
+ }
+
+ // now start the memory manager
+ final MemoryManager memoryManager;
+ try {
+ memoryManager = new MemoryManager(
+ memorySize,
+ taskManagerServicesConfiguration.getNumberOfSlots(),
+ taskManagerServicesConfiguration.getNetworkConfig().networkBufferSize(),
+ memType,
+ preAllocateMemory);
+ } catch (OutOfMemoryError e) {
+ if (memType == MemoryType.HEAP) {
+ throw new Exception("OutOfMemory error (" + e.getMessage() +
+ ") while allocating the TaskManager heap memory (" + memorySize + " bytes).", e);
+ } else if (memType == MemoryType.OFF_HEAP) {
+ throw new Exception("OutOfMemory error (" + e.getMessage() +
+ ") while allocating the TaskManager off-heap memory (" + memorySize +
+ " bytes).Try increasing the maximum direct memory (-XX:MaxDirectMemorySize)", e);
+ } else {
+ throw e;
+ }
+ }
+ return memoryManager;
+ }
+
+ /**
+ * Creates the {@link NetworkEnvironment} from the given {@link TaskManagerServicesConfiguration}.
+ *
+ * @param taskManagerServicesConfiguration to construct the network environment from
+ * @return Network environment
+ * @throws IOException
+ */
+ private static NetworkEnvironment createNetworkEnvironment(TaskManagerServicesConfiguration taskManagerServicesConfiguration) throws IOException {
+ // pre-start checks
+ checkTempDirs(taskManagerServicesConfiguration.getTmpDirPaths());
+
+ NetworkEnvironmentConfiguration networkEnvironmentConfiguration = taskManagerServicesConfiguration.getNetworkConfig();
+
+ NetworkBufferPool networkBufferPool = new NetworkBufferPool(
+ networkEnvironmentConfiguration.numNetworkBuffers(),
+ networkEnvironmentConfiguration.networkBufferSize(),
+ networkEnvironmentConfiguration.memoryType());
+
+ ConnectionManager connectionManager;
+
+ if (networkEnvironmentConfiguration.nettyConfig() != null) {
+ connectionManager = new NettyConnectionManager(networkEnvironmentConfiguration.nettyConfig());
+ } else {
+ connectionManager = new LocalConnectionManager();
+ }
+
+ ResultPartitionManager resultPartitionManager = new ResultPartitionManager();
+ TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
+
+ KvStateRegistry kvStateRegistry = new KvStateRegistry();
+
+ KvStateServer kvStateServer;
+
+ if (networkEnvironmentConfiguration.nettyConfig() != null) {
+ NettyConfig nettyConfig = networkEnvironmentConfiguration.nettyConfig();
+
+ int numNetworkThreads = networkEnvironmentConfiguration.queryServerNetworkThreads() == 0 ?
+ nettyConfig.getNumberOfSlots() : networkEnvironmentConfiguration.queryServerNetworkThreads();
+
+ int numQueryThreads = networkEnvironmentConfiguration.queryServerQueryThreads() == 0 ?
+ nettyConfig.getNumberOfSlots() : networkEnvironmentConfiguration.queryServerQueryThreads();
+
+ kvStateServer = new KvStateServer(
+ taskManagerServicesConfiguration.getTaskManagerAddress(),
+ networkEnvironmentConfiguration.queryServerPort(),
+ numNetworkThreads,
+ numQueryThreads,
+ kvStateRegistry,
+ new DisabledKvStateRequestStats());
+ } else {
+ kvStateServer = null;
+ }
+
+ // we start the network first, to make sure it can allocate its buffers first
+ final NetworkEnvironment network = new NetworkEnvironment(
+ networkBufferPool,
+ connectionManager,
+ resultPartitionManager,
+ taskEventDispatcher,
+ kvStateRegistry,
+ kvStateServer,
+ networkEnvironmentConfiguration.ioMode(),
+ networkEnvironmentConfiguration.partitionRequestInitialBackoff(),
+ networkEnvironmentConfiguration.partitinRequestMaxBackoff());
+
+ return network;
+ }
+
+ /**
+ * Validates that all the directories denoted by the strings do actually exist, are proper
+ * directories (not files), and are writable.
+ *
+ * @param tmpDirs The array of directory paths to check.
+ * @throws Exception Thrown if any of the directories does not exist or is not writable
+ * or is a file, rather than a directory.
+ */
+ private static void checkTempDirs(String[] tmpDirs) throws IOException {
+ for (String dir : tmpDirs) {
+ if (dir != null && !dir.equals("")) {
+ File file = new File(dir);
+ if (!file.exists()) {
+ throw new IOException("Temporary file directory " + file.getAbsolutePath() + " does not exist.");
+ }
+ if (!file.isDirectory()) {
+ throw new IOException("Temporary file directory " + file.getAbsolutePath() + " is not a directory.");
+ }
+ if (!file.canWrite()) {
+ throw new IOException("Temporary file directory " + file.getAbsolutePath() + " is not writable.");
+ }
+
+ if (LOG.isInfoEnabled()) {
+ long totalSpaceGb = file.getTotalSpace() >> 30;
+ long usableSpaceGb = file.getUsableSpace() >> 30;
+ double usablePercentage = (double)usableSpaceGb / totalSpaceGb * 100;
+ String path = file.getAbsolutePath();
+ LOG.info(String.format("Temporary file directory '%s': total %d GB, " + "usable %d GB (%.2f%% usable)",
+ path, totalSpaceGb, usableSpaceGb, usablePercentage));
+ }
+ } else {
+ throw new IllegalArgumentException("Temporary file directory #$id is null.");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
new file mode 100644
index 0000000..66d969a
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor;
+
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.core.memory.HeapMemorySegment;
+import org.apache.flink.core.memory.HybridMemorySegment;
+import org.apache.flink.core.memory.MemorySegmentFactory;
+import org.apache.flink.core.memory.MemoryType;
+import org.apache.flink.runtime.io.disk.iomanager.IOManager;
+import org.apache.flink.runtime.io.network.netty.NettyConfig;
+import org.apache.flink.runtime.memory.MemoryManager;
+import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
+import org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration;
+import org.apache.flink.util.MathUtils;
+
+import java.io.File;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Configuration for the task manager services such as the network environment, the memory manager,
+ * the io manager and the metric registry
+ */
+public class TaskManagerServicesConfiguration {
+
+ private final InetAddress taskManagerAddress;
+
+ private final String[] tmpDirPaths;
+
+ private final int numberOfSlots;
+
+ private final NetworkEnvironmentConfiguration networkConfig;
+
+ private final long configuredMemory;
+
+ private final boolean preAllocateMemory;
+
+ private final float memoryFraction;
+
+ private final MetricRegistryConfiguration metricRegistryConfiguration;
+
+ public TaskManagerServicesConfiguration(
+ InetAddress taskManagerAddress,
+ String[] tmpDirPaths,
+ NetworkEnvironmentConfiguration networkConfig,
+ int numberOfSlots,
+ long configuredMemory,
+ boolean preAllocateMemory,
+ float memoryFraction,
+ MetricRegistryConfiguration metricRegistryConfiguration) {
+
+ this.taskManagerAddress = checkNotNull(taskManagerAddress);
+ this.tmpDirPaths = checkNotNull(tmpDirPaths);
+ this.networkConfig = checkNotNull(networkConfig);
+ this.numberOfSlots = checkNotNull(numberOfSlots);
+
+ this.configuredMemory = configuredMemory;
+ this.preAllocateMemory = preAllocateMemory;
+ this.memoryFraction = memoryFraction;
+
+ this.metricRegistryConfiguration = checkNotNull(metricRegistryConfiguration);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Getter/Setter
+ // --------------------------------------------------------------------------------------------
+
+
+ public InetAddress getTaskManagerAddress() {
+ return taskManagerAddress;
+ }
+
+ public String[] getTmpDirPaths() {
+ return tmpDirPaths;
+ }
+
+ public NetworkEnvironmentConfiguration getNetworkConfig() { return networkConfig; }
+
+ public int getNumberOfSlots() {
+ return numberOfSlots;
+ }
+
+ public float getMemoryFraction() {
+ return memoryFraction;
+ }
+
+ public long getConfiguredMemory() {
+ return configuredMemory;
+ }
+
+ public boolean isPreAllocateMemory() {
+ return preAllocateMemory;
+ }
+
+ public MetricRegistryConfiguration getMetricRegistryConfiguration() {
+ return metricRegistryConfiguration;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Parsing of Flink configuration
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Utility method to extract TaskManager config parameters from the configuration and to
+ * sanity check them.
+ *
+ * @param configuration The configuration.
+ * @param remoteAddress identifying the IP address under which the TaskManager will be accessible
+ * @param localCommunication True, to skip initializing the network stack.
+ * Use only in cases where only one task manager runs.
+ * @return TaskExecutorConfiguration that wrappers InstanceConnectionInfo, NetworkEnvironmentConfiguration, etc.
+ */
+ public static TaskManagerServicesConfiguration fromConfiguration(
+ Configuration configuration,
+ InetAddress remoteAddress,
+ boolean localCommunication) throws Exception {
+
+ // we need this because many configs have been written with a "-1" entry
+ int slots = configuration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
+ if (slots == -1) {
+ slots = 1;
+ }
+
+ final String[] tmpDirs = configuration.getString(
+ ConfigConstants.TASK_MANAGER_TMP_DIR_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH).split(",|" + File.pathSeparator);
+
+ final NetworkEnvironmentConfiguration networkConfig = parseNetworkEnvironmentConfiguration(
+ configuration,
+ localCommunication,
+ remoteAddress,
+ slots);
+
+ // extract memory settings
+ long configuredMemory = configuration.getLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
+ checkConfigParameter(configuredMemory == -1 || configuredMemory > 0, configuredMemory,
+ ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY,
+ "MemoryManager needs at least one MB of memory. " +
+ "If you leave this config parameter empty, the system automatically " +
+ "pick a fraction of the available memory.");
+
+ boolean preAllocateMemory = configuration.getBoolean(
+ ConfigConstants.TASK_MANAGER_MEMORY_PRE_ALLOCATE_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_PRE_ALLOCATE);
+
+ float memoryFraction = configuration.getFloat(
+ ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY,
+ ConfigConstants.DEFAULT_MEMORY_MANAGER_MEMORY_FRACTION);
+ checkConfigParameter(memoryFraction > 0.0f && memoryFraction < 1.0f, memoryFraction,
+ ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY,
+ "MemoryManager fraction of the free memory must be between 0.0 and 1.0");
+
+ final MetricRegistryConfiguration metricRegistryConfiguration = MetricRegistryConfiguration.fromConfiguration(configuration);
+
+ return new TaskManagerServicesConfiguration(
+ remoteAddress,
+ tmpDirs,
+ networkConfig,
+ slots,
+ configuredMemory,
+ preAllocateMemory,
+ memoryFraction,
+ metricRegistryConfiguration);
+ }
+
+ // --------------------------------------------------------------------------
+ // Parsing and checking the TaskManager Configuration
+ // --------------------------------------------------------------------------
+
+ /**
+ * Creates the {@link NetworkEnvironmentConfiguration} from the given {@link Configuration}.
+ *
+ * @param configuration to create the network environment configuration from
+ * @param localTaskManagerCommunication true if task manager communication is local
+ * @param taskManagerAddress address of the task manager
+ * @param slots to start the task manager with
+ * @return Network environment configuration
+ */
+ private static NetworkEnvironmentConfiguration parseNetworkEnvironmentConfiguration(
+ Configuration configuration,
+ boolean localTaskManagerCommunication,
+ InetAddress taskManagerAddress,
+ int slots) throws Exception {
+
+ // ----> hosts / ports for communication and data exchange
+
+ int dataport = configuration.getInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_DATA_PORT);
+
+ checkConfigParameter(dataport > 0, dataport, ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
+ "Leave config parameter empty or use 0 to let the system choose a port automatically.");
+
+ checkConfigParameter(slots >= 1, slots, ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS,
+ "Number of task slots must be at least one.");
+
+ final int numNetworkBuffers = configuration.getInteger(
+ ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS);
+
+ checkConfigParameter(numNetworkBuffers > 0, numNetworkBuffers,
+ ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, "");
+
+ final int pageSize = configuration.getInteger(
+ ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_SEGMENT_SIZE);
+
+ // check page size of for minimum size
+ checkConfigParameter(pageSize >= MemoryManager.MIN_PAGE_SIZE, pageSize,
+ ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
+ "Minimum memory segment size is " + MemoryManager.MIN_PAGE_SIZE);
+
+ // check page size for power of two
+ checkConfigParameter(MathUtils.isPowerOf2(pageSize), pageSize,
+ ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
+ "Memory segment size must be a power of 2.");
+
+ // check whether we use heap or off-heap memory
+ final MemoryType memType;
+ if (configuration.getBoolean(ConfigConstants.TASK_MANAGER_MEMORY_OFF_HEAP_KEY, false)) {
+ memType = MemoryType.OFF_HEAP;
+ } else {
+ memType = MemoryType.HEAP;
+ }
+
+ // initialize the memory segment factory accordingly
+ if (memType == MemoryType.HEAP) {
+ if (!MemorySegmentFactory.initializeIfNotInitialized(HeapMemorySegment.FACTORY)) {
+ throw new Exception("Memory type is set to heap memory, but memory segment " +
+ "factory has been initialized for off-heap memory segments");
+ }
+ } else {
+ if (!MemorySegmentFactory.initializeIfNotInitialized(HybridMemorySegment.FACTORY)) {
+ throw new Exception("Memory type is set to off-heap memory, but memory segment " +
+ "factory has been initialized for heap memory segments");
+ }
+ }
+
+ final NettyConfig nettyConfig;
+ if (!localTaskManagerCommunication) {
+ final InetSocketAddress taskManagerInetSocketAddress = new InetSocketAddress(taskManagerAddress, dataport);
+
+ nettyConfig = new NettyConfig(taskManagerInetSocketAddress.getAddress(),
+ taskManagerInetSocketAddress.getPort(), pageSize, slots, configuration);
+ } else {
+ nettyConfig = null;
+ }
+
+ // Default spill I/O mode for intermediate results
+ final String syncOrAsync = configuration.getString(
+ ConfigConstants.TASK_MANAGER_NETWORK_DEFAULT_IO_MODE,
+ ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_DEFAULT_IO_MODE);
+
+ final IOManager.IOMode ioMode;
+ if (syncOrAsync.equals("async")) {
+ ioMode = IOManager.IOMode.ASYNC;
+ } else {
+ ioMode = IOManager.IOMode.SYNC;
+ }
+
+ final int queryServerPort = configuration.getInteger(
+ ConfigConstants.QUERYABLE_STATE_SERVER_PORT,
+ ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_PORT);
+
+ final int queryServerNetworkThreads = configuration.getInteger(
+ ConfigConstants.QUERYABLE_STATE_SERVER_NETWORK_THREADS,
+ ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_NETWORK_THREADS);
+
+ final int queryServerQueryThreads = configuration.getInteger(
+ ConfigConstants.QUERYABLE_STATE_SERVER_QUERY_THREADS,
+ ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_QUERY_THREADS);
+
+ return new NetworkEnvironmentConfiguration(
+ numNetworkBuffers,
+ pageSize,
+ memType,
+ ioMode,
+ queryServerPort,
+ queryServerNetworkThreads,
+ queryServerQueryThreads,
+ nettyConfig,
+ 500,
+ 3000);
+ }
+
+ /**
+ * Validates a condition for a config parameter and displays a standard exception, if the
+ * the condition does not hold.
+ *
+ * @param condition The condition that must hold. If the condition is false, an exception is thrown.
+ * @param parameter The parameter value. Will be shown in the exception message.
+ * @param name The name of the config parameter. Will be shown in the exception message.
+ * @param errorMessage The optional custom error message to append to the exception message.
+ */
+ private static void checkConfigParameter(
+ boolean condition,
+ Object parameter,
+ String name,
+ String errorMessage) {
+ if (!condition) {
+ throw new IllegalConfigurationException("Invalid configuration value for " + name + " : " + parameter + " - " + errorMessage);
+ }
+ }
+}
+
[46/50] [abbrv] flink git commit: Rebase fixes
Posted by tr...@apache.org.
Rebase fixes
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4c4d8eb1
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4c4d8eb1
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4c4d8eb1
Branch: refs/heads/flip-6
Commit: 4c4d8eb144f34c0d454ce7b033e29f2ab75f5cea
Parents: 09ef605
Author: Till Rohrmann <tr...@apache.org>
Authored: Mon Oct 31 19:01:22 2016 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:35 2016 +0100
----------------------------------------------------------------------
.../flink/runtime/instance/SlotDescriptor.java | 15 ++++++++-------
.../org/apache/flink/runtime/instance/SlotPool.java | 4 ++--
.../flink/runtime/taskexecutor/TaskExecutor.java | 11 ++++++-----
.../org/apache/flink/runtime/taskmanager/Task.java | 10 +++++-----
.../flink/runtime/taskmanager/TaskManager.scala | 9 +--------
.../flink/runtime/instance/AvailableSlotsTest.java | 3 ++-
.../flink/runtime/taskmanager/TaskManagerTest.java | 3 ++-
7 files changed, 26 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/4c4d8eb1/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotDescriptor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotDescriptor.java
index be7cf96..47ce422 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotDescriptor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotDescriptor.java
@@ -20,6 +20,7 @@ package org.apache.flink.runtime.instance;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -45,20 +46,20 @@ public class SlotDescriptor {
private final ResourceProfile resourceProfile;
/** TEMP until the new RPC is in place: The actor gateway to communicate with the TaskManager */
- private final ActorGateway taskManagerActorGateway;
+ private final TaskManagerGateway taskManagerGateway;
public SlotDescriptor(
final JobID jobID,
final TaskManagerLocation location,
final int slotNumber,
final ResourceProfile resourceProfile,
- final ActorGateway actorGateway)
+ final TaskManagerGateway taskManagerGateway)
{
this.jobID = checkNotNull(jobID);
this.taskManagerLocation = checkNotNull(location);
this.slotNumber = slotNumber;
this.resourceProfile = checkNotNull(resourceProfile);
- this.taskManagerActorGateway = checkNotNull(actorGateway);
+ this.taskManagerGateway = checkNotNull(taskManagerGateway);
}
public SlotDescriptor(final SlotDescriptor other) {
@@ -66,7 +67,7 @@ public class SlotDescriptor {
this.taskManagerLocation = other.taskManagerLocation;
this.slotNumber = other.slotNumber;
this.resourceProfile = other.resourceProfile;
- this.taskManagerActorGateway = other.taskManagerActorGateway;
+ this.taskManagerGateway = other.taskManagerGateway;
}
// TODO - temporary workaround until we have the SlotDesriptor in the Slot
@@ -75,7 +76,7 @@ public class SlotDescriptor {
this.taskManagerLocation = slot.getTaskManagerLocation();
this.slotNumber = slot.getRootSlotNumber();
this.resourceProfile = new ResourceProfile(0, 0);
- this.taskManagerActorGateway = slot.getTaskManagerActorGateway();
+ this.taskManagerGateway = slot.getTaskManagerGateway();
}
/**
@@ -121,8 +122,8 @@ public class SlotDescriptor {
*
* @return The actor gateway that can be used to send messages to the TaskManager.
*/
- public ActorGateway getTaskManagerActorGateway() {
- return taskManagerActorGateway;
+ public TaskManagerGateway getTaskManagerGateway() {
+ return taskManagerGateway;
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/4c4d8eb1/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
index 02166a4..44df29b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
@@ -142,7 +142,7 @@ public class SlotPool implements SlotOwner {
SimpleSlot slot = new SimpleSlot(
descriptor.getJobID(), SlotPool.this,
descriptor.getTaskManagerLocation(), descriptor.getSlotNumber(),
- descriptor.getTaskManagerActorGateway());
+ descriptor.getTaskManagerGateway());
synchronized (lock) {
// double validation since we are out of the lock protection after the slot is granted
if (registeredResources.contains(descriptor.getTaskManagerLocation().getResourceID())) {
@@ -198,7 +198,7 @@ public class SlotPool implements SlotOwner {
public SharedSlot apply(SlotDescriptor descriptor) {
SharedSlot slot = new SharedSlot(
descriptor.getJobID(), SlotPool.this, descriptor.getTaskManagerLocation(),
- descriptor.getSlotNumber(), descriptor.getTaskManagerActorGateway(),
+ descriptor.getSlotNumber(), descriptor.getTaskManagerGateway(),
sharingGroupAssignment);
synchronized (lock) {
http://git-wip-us.apache.org/repos/asf/flink/blob/4c4d8eb1/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index 5eb8e6a..071e706 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -853,11 +853,12 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
jobMasterLeaderId,
jobMasterGateway,
new TaskExecutionState(
- task.getJobID(),
- task.getExecutionId(),
- task.getExecutionState(),
- task.getFailureCause(),
- accumulatorSnapshot));
+ task.getJobID(),
+ task.getExecutionId(),
+ task.getExecutionState(),
+ task.getFailureCause(),
+ accumulatorSnapshot,
+ task.getMetricGroup().getIOMetricGroup().createSnapshot()));
} else {
log.error("Cannot find task with ID {} to unregister.", executionAttemptID);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4c4d8eb1/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index 7e9fbfa..eeba532 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -929,7 +929,7 @@ public class Task implements Runnable, TaskActions {
taskNameWithSubtask,
taskCancellationInterval,
taskCancellationTimeout,
- taskManagerConnection,
+ taskManagerActions,
producedPartitions,
inputGates);
Thread cancelThread = new Thread(executingThread.getThreadGroup(), canceler,
@@ -1229,7 +1229,7 @@ public class Task implements Runnable, TaskActions {
private final long interruptTimeout;
/** TaskManager to notify about a timeout */
- private final TaskManagerConnection taskManager;
+ private final TaskManagerActions taskManagerActions;
/** Watch Dog thread */
@Nullable
@@ -1242,7 +1242,7 @@ public class Task implements Runnable, TaskActions {
String taskName,
long cancellationInterval,
long cancellationTimeout,
- TaskManagerConnection taskManager,
+ TaskManagerActions taskManagerActions,
ResultPartition[] producedPartitions,
SingleInputGate[] inputGates) {
@@ -1252,7 +1252,7 @@ public class Task implements Runnable, TaskActions {
this.taskName = taskName;
this.interruptInterval = cancellationInterval;
this.interruptTimeout = cancellationTimeout;
- this.taskManager = taskManager;
+ this.taskManagerActions = taskManagerActions;
this.producedPartitions = producedPartitions;
this.inputGates = inputGates;
@@ -1367,7 +1367,7 @@ public class Task implements Runnable, TaskActions {
duration,
bld.toString());
- taskManager.notifyFatalError(msg, null);
+ taskManagerActions.notifyFatalError(msg, null);
return; // done, don't forget to leave the loop
} else {
http://git-wip-us.apache.org/repos/asf/flink/blob/4c4d8eb1/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 5ecfb80..f7c462f 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -22,6 +22,7 @@ import java.io.{File, FileInputStream, IOException}
import java.lang.management.ManagementFactory
import java.net.{InetAddress, InetSocketAddress}
import java.util
+import java.util.concurrent.TimeUnit
import java.util.{Collections, UUID}
import _root_.akka.actor._
@@ -1961,12 +1962,4 @@ object TaskManager {
(protocol, hostname, port)
}
-
-
- // --------------------------------------------------------------------------
- // Miscellaneous Utilities
- // --------------------------------------------------------------------------
-
- /**
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4c4d8eb1/flink-runtime/src/test/java/org/apache/flink/runtime/instance/AvailableSlotsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/AvailableSlotsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/AvailableSlotsTest.java
index 872810f..8e31085 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/AvailableSlotsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/AvailableSlotsTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.instance;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.junit.Test;
@@ -118,6 +119,6 @@ public class AvailableSlotsTest {
{
TaskManagerLocation location = mock(TaskManagerLocation.class);
when(location.getResourceID()).thenReturn(resourceID);
- return new SlotDescriptor(jobID, location, slotNumber, resourceProfile, mock(ActorGateway.class));
+ return new SlotDescriptor(jobID, location, slotNumber, resourceProfile, mock(TaskManagerGateway.class));
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4c4d8eb1/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
index 369e6d1..06add39 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
@@ -1429,7 +1429,8 @@ public class TaskManagerTest extends TestLogger {
false // don't deploy eagerly but with the first completed memory buffer
);
- final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(jid, "TestJob", vid, eid, executionConfig,
+ final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
+ jid, new AllocationID(), "TestJob", vid, eid, executionConfig,
"TestTask", 1, 0, 1, 0, new Configuration(), new Configuration(),
TestInvokableRecordCancel.class.getName(),
Collections.singletonList(resultPartitionDeploymentDescriptor),
[41/50] [abbrv] flink git commit: [hotfix] Add a DefaultSlotManager
similar to the TestingSlotManager
Posted by tr...@apache.org.
[hotfix] Add a DefaultSlotManager similar to the TestingSlotManager
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e61ade68
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e61ade68
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e61ade68
Branch: refs/heads/flip-6
Commit: e61ade6881de858959c1f74d2d13ed07dabd5fda
Parents: 73dafd4
Author: Stephan Ewen <se...@apache.org>
Authored: Mon Oct 17 16:38:17 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:33 2016 +0100
----------------------------------------------------------------------
.../ResourceManagerServices.java | 2 +-
.../slotmanager/DefaultSlotManager.java | 69 ++++++++++++++++++++
2 files changed, 70 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e61ade68/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
index 16d0a7d..c524604 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
@@ -38,7 +38,7 @@ public interface ResourceManagerServices {
void allocateResource(ResourceProfile resourceProfile);
/**
- * Gets the async excutor which executes outside of the main thread of the ResourceManager
+ * Gets the async executor which executes outside of the main thread of the ResourceManager
*/
Executor getAsyncExecutor();
http://git-wip-us.apache.org/repos/asf/flink/blob/e61ade68/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/DefaultSlotManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/DefaultSlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/DefaultSlotManager.java
new file mode 100644
index 0000000..9508936
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/DefaultSlotManager.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.slotmanager;
+
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceSlot;
+import org.apache.flink.runtime.clusterframework.types.SlotID;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerServices;
+import org.apache.flink.runtime.resourcemanager.SlotRequest;
+
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * A slot manager that answers requests with slots without any special logic. The first slot
+ * in the maps to match a request is chosen.
+ */
+public class DefaultSlotManager extends SlotManager {
+
+ public DefaultSlotManager(ResourceManagerServices rmServices) {
+ super(rmServices);
+ }
+
+ @Override
+ protected ResourceSlot chooseSlotToUse(SlotRequest request, Map<SlotID, ResourceSlot> freeSlots) {
+ final Iterator<ResourceSlot> slotIterator = freeSlots.values().iterator();
+ if (slotIterator.hasNext()) {
+ return slotIterator.next();
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ protected SlotRequest chooseRequestToFulfill(ResourceSlot offeredSlot, Map<AllocationID, SlotRequest> pendingRequests) {
+ final Iterator<SlotRequest> requestIterator = pendingRequests.values().iterator();
+ if (requestIterator.hasNext()) {
+ return requestIterator.next();
+ } else {
+ return null;
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ public static class Factory implements SlotManagerFactory {
+
+ @Override
+ public SlotManager create(ResourceManagerServices rmServices) {
+ return new DefaultSlotManager(rmServices);
+ }
+ }
+}
[22/50] [abbrv] flink git commit: [FLINK-4738] [TaskManager] Port
TaskManager logic to new Flip-6 TaskManager
Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/PartitionException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/PartitionException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/PartitionException.java
new file mode 100644
index 0000000..eecd0ae
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/PartitionException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.exceptions;
+
+import org.apache.flink.runtime.taskexecutor.TaskExecutor;
+
+/**
+ * Exception indicating a problem with the result partitions on the {@link TaskExecutor} side.
+ */
+public class PartitionException extends TaskManagerException {
+
+ private static final long serialVersionUID = 6248696963418276618L;
+
+ public PartitionException(String message) {
+ super(message);
+ }
+
+ public PartitionException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public PartitionException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/TaskException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/TaskException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/TaskException.java
new file mode 100644
index 0000000..a4a89c2
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/TaskException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.exceptions;
+
+import org.apache.flink.runtime.taskexecutor.TaskExecutor;
+
+/**
+ * Exception indicating a task related problem on the {@link TaskExecutor}.
+ */
+public class TaskException extends TaskManagerException {
+
+ private static final long serialVersionUID = 968001398103156856L;
+
+ public TaskException(String message) {
+ super(message);
+ }
+
+ public TaskException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public TaskException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/TaskManagerException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/TaskManagerException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/TaskManagerException.java
new file mode 100644
index 0000000..62d186e
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/TaskManagerException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.exceptions;
+
+import org.apache.flink.runtime.taskexecutor.TaskExecutor;
+
+/**
+ * Base exception thrown by the {@link TaskExecutor}.
+ */
+public class TaskManagerException extends Exception {
+
+ private static final long serialVersionUID = -2997745772227694731L;
+
+ public TaskManagerException(String message) {
+ super(message);
+ }
+
+ public TaskManagerException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public TaskManagerException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/TaskSubmissionException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/TaskSubmissionException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/TaskSubmissionException.java
new file mode 100644
index 0000000..23f7812
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/TaskSubmissionException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.exceptions;
+
+import org.apache.flink.runtime.taskmanager.Task;
+
+/**
+ * Exception indicating a problem with the {@link Task} submission at the {@link TaskException}.
+ */
+public class TaskSubmissionException extends TaskManagerException {
+
+ private static final long serialVersionUID = 4589813591317690486L;
+
+ public TaskSubmissionException(String message) {
+ super(message);
+ }
+
+ public TaskSubmissionException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public TaskSubmissionException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java
new file mode 100644
index 0000000..246c11d
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.rpc;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.checkpoint.CheckpointCoordinatorGateway;
+import org.apache.flink.runtime.checkpoint.CheckpointMetaData;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.state.CheckpointStateHandles;
+import org.apache.flink.runtime.taskmanager.CheckpointResponder;
+import org.apache.flink.util.Preconditions;
+
+public class RpcCheckpointResponder implements CheckpointResponder {
+
+ private final CheckpointCoordinatorGateway checkpointCoordinatorGateway;
+
+ public RpcCheckpointResponder(CheckpointCoordinatorGateway checkpointCoordinatorGateway) {
+ this.checkpointCoordinatorGateway = Preconditions.checkNotNull(checkpointCoordinatorGateway);
+ }
+
+ @Override
+ public void acknowledgeCheckpoint(
+ JobID jobID,
+ ExecutionAttemptID executionAttemptID,
+ CheckpointMetaData checkpointMetaData,
+ CheckpointStateHandles checkpointStateHandles) {
+
+ checkpointCoordinatorGateway.acknowledgeCheckpoint(
+ jobID,
+ executionAttemptID,
+ checkpointMetaData,
+ checkpointStateHandles);
+
+ }
+
+ @Override
+ public void declineCheckpoint(JobID jobID, ExecutionAttemptID executionAttemptID, CheckpointMetaData checkpoint) {
+ checkpointCoordinatorGateway.declineCheckpoint(jobID, executionAttemptID, checkpoint);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcInputSplitProvider.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcInputSplitProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcInputSplitProvider.java
new file mode 100644
index 0000000..4850d63
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcInputSplitProvider.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.rpc;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.core.io.InputSplit;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
+import org.apache.flink.runtime.jobgraph.tasks.InputSplitProviderException;
+import org.apache.flink.runtime.jobmaster.JobMasterGateway;
+import org.apache.flink.runtime.jobmaster.SerializedInputSplit;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.Preconditions;
+
+public class RpcInputSplitProvider implements InputSplitProvider {
+ private final JobMasterGateway jobMasterGateway;
+ private final JobID jobID;
+ private final JobVertexID jobVertexID;
+ private final ExecutionAttemptID executionAttemptID;
+ private final Time timeout;
+
+ public RpcInputSplitProvider(
+ JobMasterGateway jobMasterGateway,
+ JobID jobID,
+ JobVertexID jobVertexID,
+ ExecutionAttemptID executionAttemptID,
+ Time timeout) {
+ this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
+ this.jobID = Preconditions.checkNotNull(jobID);
+ this.jobVertexID = Preconditions.checkNotNull(jobVertexID);
+ this.executionAttemptID = Preconditions.checkNotNull(executionAttemptID);
+ this.timeout = Preconditions.checkNotNull(timeout);
+ }
+
+
+ @Override
+ public InputSplit getNextInputSplit(ClassLoader userCodeClassLoader) throws InputSplitProviderException {
+ Preconditions.checkNotNull(userCodeClassLoader);
+
+ Future<SerializedInputSplit> futureInputSplit = jobMasterGateway.requestNextInputSplit(jobVertexID, executionAttemptID);
+
+ try {
+ SerializedInputSplit serializedInputSplit = futureInputSplit.get(timeout.getSize(), timeout.getUnit());
+
+ if (serializedInputSplit.isEmpty()) {
+ return null;
+ } else {
+ return InstantiationUtil.deserializeObject(serializedInputSplit.getInputSplitData(), userCodeClassLoader);
+ }
+ } catch (Exception e) {
+ throw new InputSplitProviderException("Requesting the next input split failed.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcKvStateRegistryListener.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcKvStateRegistryListener.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcKvStateRegistryListener.java
new file mode 100644
index 0000000..3692a71
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcKvStateRegistryListener.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.rpc;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.query.KvStateID;
+import org.apache.flink.runtime.query.KvStateRegistryGateway;
+import org.apache.flink.runtime.query.KvStateRegistryListener;
+import org.apache.flink.runtime.query.KvStateServerAddress;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.util.Preconditions;
+
+public class RpcKvStateRegistryListener implements KvStateRegistryListener {
+
+ private final KvStateRegistryGateway kvStateRegistryGateway;
+ private final KvStateServerAddress kvStateServerAddress;
+
+ public RpcKvStateRegistryListener(
+ KvStateRegistryGateway kvStateRegistryGateway,
+ KvStateServerAddress kvStateServerAddress) {
+ this.kvStateRegistryGateway = Preconditions.checkNotNull(kvStateRegistryGateway);
+ this.kvStateServerAddress = Preconditions.checkNotNull(kvStateServerAddress);
+ }
+
+ @Override
+ public void notifyKvStateRegistered(
+ JobID jobId,
+ JobVertexID jobVertexId,
+ KeyGroupRange keyGroupRange,
+ String registrationName,
+ KvStateID kvStateId) {
+ kvStateRegistryGateway.notifyKvStateRegistered(
+ jobId,
+ jobVertexId,
+ keyGroupRange,
+ registrationName,
+ kvStateId,
+ kvStateServerAddress);
+
+ }
+
+ @Override
+ public void notifyKvStateUnregistered(
+ JobID jobId,
+ JobVertexID jobVertexId,
+ KeyGroupRange keyGroupRange,
+ String registrationName) {
+
+ kvStateRegistryGateway.notifyKvStateUnregistered(
+ jobId,
+ jobVertexId,
+ keyGroupRange,
+ registrationName);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcPartitionStateChecker.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcPartitionStateChecker.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcPartitionStateChecker.java
new file mode 100644
index 0000000..ab111ad
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcPartitionStateChecker.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.rpc;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.io.network.PartitionState;
+import org.apache.flink.runtime.io.network.netty.PartitionStateChecker;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
+import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
+import org.apache.flink.runtime.jobmaster.JobMasterGateway;
+import org.apache.flink.util.Preconditions;
+
+public class RpcPartitionStateChecker implements PartitionStateChecker {
+
+ private final JobMasterGateway jobMasterGateway;
+
+ public RpcPartitionStateChecker(JobMasterGateway jobMasterGateway) {
+ this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
+ }
+
+ @Override
+ public Future<PartitionState> requestPartitionState(
+ JobID jobId,
+ ExecutionAttemptID executionId,
+ IntermediateDataSetID resultId,
+ ResultPartitionID partitionId) {
+
+ return jobMasterGateway.requestPartitionState(partitionId, executionId, resultId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcResultPartitionConsumableNotifier.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcResultPartitionConsumableNotifier.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcResultPartitionConsumableNotifier.java
new file mode 100644
index 0000000..29ad3b6
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcResultPartitionConsumableNotifier.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.rpc;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.concurrent.ApplyFunction;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionConsumableNotifier;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
+import org.apache.flink.runtime.jobmaster.JobMasterGateway;
+import org.apache.flink.runtime.messages.Acknowledge;
+import org.apache.flink.runtime.taskmanager.TaskActions;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.Executor;
+
+public class RpcResultPartitionConsumableNotifier implements ResultPartitionConsumableNotifier {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RpcResultPartitionConsumableNotifier.class);
+
+ private final JobMasterGateway jobMasterGateway;
+ private final Executor executor;
+ private final Time timeout;
+
+ public RpcResultPartitionConsumableNotifier(
+ JobMasterGateway jobMasterGateway,
+ Executor executor,
+ Time timeout) {
+ this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
+ this.executor = Preconditions.checkNotNull(executor);
+ this.timeout = Preconditions.checkNotNull(timeout);
+ }
+ @Override
+ public void notifyPartitionConsumable(JobID jobId, ResultPartitionID partitionId, final TaskActions taskActions) {
+ Future<Acknowledge> acknowledgeFuture = jobMasterGateway.scheduleOrUpdateConsumers(partitionId, timeout);
+
+ acknowledgeFuture.exceptionallyAsync(new ApplyFunction<Throwable, Void>() {
+ @Override
+ public Void apply(Throwable value) {
+ LOG.error("Could not schedule or update consumers at the JobManager.", value);
+
+ taskActions.failExternally(new RuntimeException("Could not notify JobManager to schedule or update consumers.", value));
+
+ return null;
+ }
+ }, executor);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/utils/TaskExecutorMetricsInitializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/utils/TaskExecutorMetricsInitializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/utils/TaskExecutorMetricsInitializer.java
new file mode 100644
index 0000000..1f8d5ed
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/utils/TaskExecutorMetricsInitializer.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.utils;
+
+import com.sun.management.OperatingSystemMXBean;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.io.network.NetworkEnvironment;
+import org.apache.flink.runtime.taskexecutor.TaskExecutor;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.management.AttributeNotFoundException;
+import javax.management.InstanceNotFoundException;
+import javax.management.MBeanException;
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import javax.management.ReflectionException;
+import java.lang.management.ClassLoadingMXBean;
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.ThreadMXBean;
+import java.util.List;
+
+/**
+ * Utility class ot initialize {@link TaskExecutor} specific metrics.
+ */
+public class TaskExecutorMetricsInitializer {
+ private static final Logger LOG = LoggerFactory.getLogger(TaskExecutorMetricsInitializer.class);
+
+ public static void instantiateStatusMetrics(
+ MetricGroup taskManagerMetricGroup,
+ NetworkEnvironment network) {
+ MetricGroup status = taskManagerMetricGroup.addGroup("Status");
+
+ instantiateNetworkMetrics(status.addGroup("Network"), network);
+
+ MetricGroup jvm = status.addGroup("JVM");
+
+ instantiateClassLoaderMetrics(jvm.addGroup("ClassLoader"));
+ instantiateGarbageCollectorMetrics(jvm.addGroup("GarbageCollector"));
+ instantiateMemoryMetrics(jvm.addGroup("Memory"));
+ instantiateThreadMetrics(jvm.addGroup("Threads"));
+ instantiateCPUMetrics(jvm.addGroup("CPU"));
+ }
+
+ private static void instantiateNetworkMetrics(
+ MetricGroup metrics,
+ final NetworkEnvironment network) {
+ metrics.<Long, Gauge<Long>>gauge("TotalMemorySegments", new Gauge<Long> () {
+ @Override
+ public Long getValue() {
+ return (long) network.getNetworkBufferPool().getTotalNumberOfMemorySegments();
+ }
+ });
+
+ metrics.<Long, Gauge<Long>>gauge("AvailableMemorySegments", new Gauge<Long> () {
+ @Override
+ public Long getValue() {
+ return (long) network.getNetworkBufferPool().getNumberOfAvailableMemorySegments();
+ }
+ });
+ }
+
+ private static void instantiateClassLoaderMetrics(MetricGroup metrics) {
+ final ClassLoadingMXBean mxBean = ManagementFactory.getClassLoadingMXBean();
+
+ metrics.<Long, Gauge<Long>>gauge("ClassesLoaded", new Gauge<Long> () {
+ @Override
+ public Long getValue() {
+ return mxBean.getTotalLoadedClassCount();
+ }
+ });
+
+ metrics.<Long, Gauge<Long>>gauge("ClassesUnloaded", new Gauge<Long> () {
+ @Override
+ public Long getValue() {
+ return mxBean.getUnloadedClassCount();
+ }
+ });
+ }
+
+ private static void instantiateGarbageCollectorMetrics(MetricGroup metrics) {
+ List<GarbageCollectorMXBean> garbageCollectors = ManagementFactory.getGarbageCollectorMXBeans();
+
+ for (final GarbageCollectorMXBean garbageCollector: garbageCollectors) {
+ MetricGroup gcGroup = metrics.addGroup(garbageCollector.getName());
+
+ gcGroup.<Long, Gauge<Long>>gauge("Count", new Gauge<Long> () {
+ @Override
+ public Long getValue() {
+ return garbageCollector.getCollectionCount();
+ }
+ });
+
+ gcGroup.<Long, Gauge<Long>>gauge("Time", new Gauge<Long> () {
+ @Override
+ public Long getValue() {
+ return garbageCollector.getCollectionTime();
+ }
+ });
+ }
+ }
+
+ private static void instantiateMemoryMetrics(MetricGroup metrics) {
+ final MemoryMXBean mxBean = ManagementFactory.getMemoryMXBean();
+
+ MetricGroup heap = metrics.addGroup("Heap");
+
+ heap.<Long, Gauge<Long>>gauge("Used", new Gauge<Long> () {
+ @Override
+ public Long getValue() {
+ return mxBean.getHeapMemoryUsage().getUsed();
+ }
+ });
+ heap.<Long, Gauge<Long>>gauge("Committed", new Gauge<Long> () {
+ @Override
+ public Long getValue() {
+ return mxBean.getHeapMemoryUsage().getCommitted();
+ }
+ });
+ heap.<Long, Gauge<Long>>gauge("Max", new Gauge<Long> () {
+ @Override
+ public Long getValue() {
+ return mxBean.getHeapMemoryUsage().getMax();
+ }
+ });
+
+ MetricGroup nonHeap = metrics.addGroup("NonHeap");
+
+ nonHeap.<Long, Gauge<Long>>gauge("Used", new Gauge<Long> () {
+ @Override
+ public Long getValue() {
+ return mxBean.getNonHeapMemoryUsage().getUsed();
+ }
+ });
+ nonHeap.<Long, Gauge<Long>>gauge("Committed", new Gauge<Long> () {
+ @Override
+ public Long getValue() {
+ return mxBean.getNonHeapMemoryUsage().getCommitted();
+ }
+ });
+ nonHeap.<Long, Gauge<Long>>gauge("Max", new Gauge<Long> () {
+ @Override
+ public Long getValue() {
+ return mxBean.getNonHeapMemoryUsage().getMax();
+ }
+ });
+
+ final MBeanServer con = ManagementFactory.getPlatformMBeanServer();
+
+ final String directBufferPoolName = "java.nio:type=BufferPool,name=direct";
+
+ try {
+ final ObjectName directObjectName = new ObjectName(directBufferPoolName);
+
+ MetricGroup direct = metrics.addGroup("Direct");
+
+ direct.<Long, Gauge<Long>>gauge("Count", new TaskExecutorMetricsInitializer.AttributeGauge<>(con, directObjectName, "Count", -1L));
+ direct.<Long, Gauge<Long>>gauge("MemoryUsed", new TaskExecutorMetricsInitializer.AttributeGauge<>(con, directObjectName, "MemoryUsed", -1L));
+ direct.<Long, Gauge<Long>>gauge("TotalCapacity", new TaskExecutorMetricsInitializer.AttributeGauge<>(con, directObjectName, "TotalCapacity", -1L));
+ } catch (MalformedObjectNameException e) {
+ LOG.warn("Could not create object name {}.", directBufferPoolName, e);
+ }
+
+ final String mappedBufferPoolName = "java.nio:type=BufferPool,name=mapped";
+
+ try {
+ final ObjectName mappedObjectName = new ObjectName(mappedBufferPoolName);
+
+ MetricGroup mapped = metrics.addGroup("Mapped");
+
+ mapped.<Long, Gauge<Long>>gauge("Count", new TaskExecutorMetricsInitializer.AttributeGauge<>(con, mappedObjectName, "Count", -1L));
+ mapped.<Long, Gauge<Long>>gauge("MemoryUsed", new TaskExecutorMetricsInitializer.AttributeGauge<>(con, mappedObjectName, "MemoryUsed", -1L));
+ mapped.<Long, Gauge<Long>>gauge("TotalCapacity", new TaskExecutorMetricsInitializer.AttributeGauge<>(con, mappedObjectName, "TotalCapacity", -1L));
+ } catch (MalformedObjectNameException e) {
+ LOG.warn("Could not create object name {}.", mappedBufferPoolName, e);
+ }
+ }
+
+ private static void instantiateThreadMetrics(MetricGroup metrics) {
+ final ThreadMXBean mxBean = ManagementFactory.getThreadMXBean();
+
+ metrics.<Integer, Gauge<Integer>>gauge("Count", new Gauge<Integer> () {
+ @Override
+ public Integer getValue() {
+ return mxBean.getThreadCount();
+ }
+ });
+ }
+
+ private static void instantiateCPUMetrics(MetricGroup metrics) {
+ try {
+ final OperatingSystemMXBean mxBean = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
+
+ metrics.<Double, Gauge<Double>>gauge("Load", new Gauge<Double> () {
+ @Override
+ public Double getValue() {
+ return mxBean.getProcessCpuLoad();
+ }
+ });
+ metrics.<Long, Gauge<Long>>gauge("Time", new Gauge<Long> () {
+ @Override
+ public Long getValue() {
+ return mxBean.getProcessCpuTime();
+ }
+ });
+ } catch (Exception e) {
+ LOG.warn("Cannot access com.sun.management.OperatingSystemMXBean.getProcessCpuLoad()" +
+ " - CPU load metrics will not be available.", e);
+ }
+ }
+
+ private static final class AttributeGauge<T> implements Gauge<T> {
+ private final MBeanServer server;
+ private final ObjectName objectName;
+ private final String attributeName;
+ private final T errorValue;
+
+ private AttributeGauge(MBeanServer server, ObjectName objectName, String attributeName, T errorValue) {
+ this.server = Preconditions.checkNotNull(server);
+ this.objectName = Preconditions.checkNotNull(objectName);
+ this.attributeName = Preconditions.checkNotNull(attributeName);
+ this.errorValue = errorValue;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public T getValue() {
+ try {
+ return (T) server.getAttribute(objectName, attributeName);
+ } catch (MBeanException | AttributeNotFoundException | InstanceNotFoundException | ReflectionException e) {
+ LOG.warn("Could not read attribute {}.", attributeName, e);
+ return errorValue;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/ActorGatewayTaskManagerActions.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/ActorGatewayTaskManagerActions.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/ActorGatewayTaskManagerActions.java
new file mode 100644
index 0000000..b3a0cbb
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/ActorGatewayTaskManagerActions.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskmanager;
+
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.instance.ActorGateway;
+import org.apache.flink.runtime.messages.TaskManagerMessages;
+import org.apache.flink.runtime.messages.TaskMessages;
+import org.apache.flink.util.Preconditions;
+
+/**
+ * Implementation using {@link ActorGateway} to forward the messages.
+ */
+public class ActorGatewayTaskManagerActions implements TaskManagerActions {
+
+ private final ActorGateway actorGateway;
+
+ public ActorGatewayTaskManagerActions(ActorGateway actorGateway) {
+ this.actorGateway = Preconditions.checkNotNull(actorGateway);
+ }
+
+ @Override
+ public void notifyFinalState(ExecutionAttemptID executionAttemptID) {
+ actorGateway.tell(new TaskMessages.TaskInFinalState(executionAttemptID));
+ }
+
+ @Override
+ public void notifyFatalError(String message, Throwable cause) {
+ actorGateway.tell(new TaskManagerMessages.FatalError(message, cause));
+ }
+
+ @Override
+ public void failTask(ExecutionAttemptID executionAttemptID, Throwable cause) {
+ actorGateway.tell(new TaskMessages.FailTask(executionAttemptID, cause));
+ }
+
+ @Override
+ public void updateTaskExecutionState(TaskExecutionState taskExecutionState) {
+ TaskMessages.UpdateTaskExecutionState actorMessage = new TaskMessages.UpdateTaskExecutionState(taskExecutionState);
+
+ actorGateway.tell(actorMessage);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/ActorGatewayTaskManagerConnection.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/ActorGatewayTaskManagerConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/ActorGatewayTaskManagerConnection.java
deleted file mode 100644
index cddac55..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/ActorGatewayTaskManagerConnection.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.taskmanager;
-
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
-import org.apache.flink.runtime.instance.ActorGateway;
-import org.apache.flink.runtime.messages.TaskManagerMessages;
-import org.apache.flink.runtime.messages.TaskMessages;
-import org.apache.flink.util.Preconditions;
-
-/**
- * Implementation using {@link ActorGateway} to forward the messages.
- */
-public class ActorGatewayTaskManagerConnection implements TaskManagerConnection {
-
- private final ActorGateway actorGateway;
-
- public ActorGatewayTaskManagerConnection(ActorGateway actorGateway) {
- this.actorGateway = Preconditions.checkNotNull(actorGateway);
- }
-
- @Override
- public void notifyFinalState(ExecutionAttemptID executionAttemptID) {
- actorGateway.tell(new TaskMessages.TaskInFinalState(executionAttemptID));
- }
-
- @Override
- public void notifyFatalError(String message, Throwable cause) {
- actorGateway.tell(new TaskManagerMessages.FatalError(message, cause));
- }
-
- @Override
- public void failTask(ExecutionAttemptID executionAttemptID, Throwable cause) {
- actorGateway.tell(new TaskMessages.FailTask(executionAttemptID, cause));
- }
-
- @Override
- public void updateTaskExecutionState(TaskExecutionState taskExecutionState) {
- TaskMessages.UpdateTaskExecutionState actorMessage = new TaskMessages.UpdateTaskExecutionState(taskExecutionState);
-
- actorGateway.tell(actorMessage);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index 3a5c96f..0bd3a89 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -27,6 +27,7 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.TaskManagerOptions;
import org.apache.flink.core.fs.Path;
import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
@@ -131,6 +132,9 @@ public class Task implements Runnable, TaskActions {
/** The execution attempt of the parallel subtask */
private final ExecutionAttemptID executionId;
+ /** ID which identifies the slot in which the task is supposed to run */
+ private final AllocationID allocationID;
+
/** TaskInfo object for this task */
private final TaskInfo taskInfo;
@@ -176,7 +180,7 @@ public class Task implements Runnable, TaskActions {
private final Map<IntermediateDataSetID, SingleInputGate> inputGatesById;
/** Connection to the task manager */
- private final TaskManagerConnection taskManagerConnection;
+ private final TaskManagerActions taskManagerActions;
/** Input split provider for the task */
private final InputSplitProvider inputSplitProvider;
@@ -254,7 +258,7 @@ public class Task implements Runnable, TaskActions {
IOManager ioManager,
NetworkEnvironment networkEnvironment,
BroadcastVariableManager bcVarManager,
- TaskManagerConnection taskManagerConnection,
+ TaskManagerActions taskManagerActions,
InputSplitProvider inputSplitProvider,
CheckpointResponder checkpointResponder,
LibraryCacheManager libraryCache,
@@ -269,6 +273,7 @@ public class Task implements Runnable, TaskActions {
this.jobId = checkNotNull(tdd.getJobID());
this.vertexId = checkNotNull(tdd.getVertexID());
this.executionId = checkNotNull(tdd.getExecutionId());
+ this.allocationID = checkNotNull(tdd.getAllocationID());
this.taskNameWithSubtask = taskInfo.getTaskNameWithSubtasks();
this.jobConfiguration = checkNotNull(tdd.getJobConfiguration());
this.taskConfiguration = checkNotNull(tdd.getTaskConfiguration());
@@ -289,7 +294,7 @@ public class Task implements Runnable, TaskActions {
this.inputSplitProvider = checkNotNull(inputSplitProvider);
this.checkpointResponder = checkNotNull(checkpointResponder);
- this.taskManagerConnection = checkNotNull(taskManagerConnection);
+ this.taskManagerActions = checkNotNull(taskManagerActions);
this.libraryCache = checkNotNull(libraryCache);
this.fileCache = checkNotNull(fileCache);
@@ -382,6 +387,10 @@ public class Task implements Runnable, TaskActions {
return executionId;
}
+ public AllocationID getAllocationID() {
+ return allocationID;
+ }
+
public TaskInfo getTaskInfo() {
return taskInfo;
}
@@ -619,7 +628,7 @@ public class Task implements Runnable, TaskActions {
// notify everyone that we switched to running
notifyObservers(ExecutionState.RUNNING, null);
- taskManagerConnection.updateTaskExecutionState(new TaskExecutionState(jobId, executionId, ExecutionState.RUNNING));
+ taskManagerActions.updateTaskExecutionState(new TaskExecutionState(jobId, executionId, ExecutionState.RUNNING));
// make sure the user code classloader is accessible thread-locally
executingThread.setContextClassLoader(userCodeClassLoader);
@@ -812,11 +821,11 @@ public class Task implements Runnable, TaskActions {
}
private void notifyFinalState() {
- taskManagerConnection.notifyFinalState(executionId);
+ taskManagerActions.notifyFinalState(executionId);
}
private void notifyFatalError(String message, Throwable cause) {
- taskManagerConnection.notifyFatalError(message, cause);
+ taskManagerActions.notifyFatalError(message, cause);
}
// ----------------------------------------------------------------------------------------------------------------
@@ -842,7 +851,7 @@ public class Task implements Runnable, TaskActions {
((StoppableTask)Task.this.invokable).stop();
} catch(RuntimeException e) {
LOG.error("Stopping task " + taskNameWithSubtask + " failed.", e);
- taskManagerConnection.failTask(executionId, e);
+ taskManagerActions.failTask(executionId, e);
}
}
};
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java
index 7bea5d3..f246cc2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java
@@ -25,6 +25,8 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.executiongraph.IOMetrics;
import org.apache.flink.runtime.util.SerializedThrowable;
+import java.io.Serializable;
+
/**
* This class represents an update about a task's execution state.
*
@@ -35,7 +37,7 @@ import org.apache.flink.runtime.util.SerializedThrowable;
* exception field transient and deserialized it lazily, with the
* appropriate class loader.
*/
-public class TaskExecutionState implements java.io.Serializable {
+public class TaskExecutionState implements Serializable {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskInputSplitProvider.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskInputSplitProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskInputSplitProvider.java
index 60beae0..31c518a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskInputSplitProvider.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskInputSplitProvider.java
@@ -24,6 +24,7 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.instance.ActorGateway;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
+import org.apache.flink.runtime.jobgraph.tasks.InputSplitProviderException;
import org.apache.flink.runtime.messages.JobManagerMessages;
import org.apache.flink.util.InstantiationUtil;
@@ -63,35 +64,45 @@ public class TaskInputSplitProvider implements InputSplitProvider {
}
@Override
- public InputSplit getNextInputSplit(ClassLoader userCodeClassLoader) {
+ public InputSplit getNextInputSplit(ClassLoader userCodeClassLoader) throws InputSplitProviderException {
Preconditions.checkNotNull(userCodeClassLoader);
+ final Future<Object> response = jobManager.ask(
+ new JobManagerMessages.RequestNextInputSplit(jobID, vertexID, executionID),
+ timeout);
+
+ final Object result;
+
try {
- final Future<Object> response = jobManager.ask(
- new JobManagerMessages.RequestNextInputSplit(jobID, vertexID, executionID),
- timeout);
+ result = Await.result(response, timeout);
+ } catch (Exception e) {
+ throw new InputSplitProviderException("Did not receive next input split from JobManager.", e);
+ }
- final Object result = Await.result(response, timeout);
+ if(result instanceof JobManagerMessages.NextInputSplit){
+ final JobManagerMessages.NextInputSplit nextInputSplit =
+ (JobManagerMessages.NextInputSplit) result;
- if(result instanceof JobManagerMessages.NextInputSplit){
- final JobManagerMessages.NextInputSplit nextInputSplit =
- (JobManagerMessages.NextInputSplit) result;
+ byte[] serializedData = nextInputSplit.splitData();
- byte[] serializedData = nextInputSplit.splitData();
+ if(serializedData == null) {
+ return null;
+ } else {
+ final Object deserialized;
- if(serializedData == null) {
- return null;
- } else {
- Object deserialized = InstantiationUtil.deserializeObject(serializedData,
+ try {
+ deserialized = InstantiationUtil.deserializeObject(serializedData,
userCodeClassLoader);
- return (InputSplit) deserialized;
+ } catch (Exception e) {
+ throw new InputSplitProviderException("Could not deserialize the serialized input split.", e);
}
- } else {
- throw new Exception("RequestNextInputSplit requires a response of type " +
- "NextInputSplit. Instead response is of type " + result.getClass() + '.');
+
+ return (InputSplit) deserialized;
}
- } catch (Exception e) {
- throw new RuntimeException("Requesting the next InputSplit failed.", e);
+ } else {
+ throw new InputSplitProviderException("RequestNextInputSplit requires a response of type " +
+ "NextInputSplit. Instead response is of type " + result.getClass() + '.');
}
+
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerActions.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerActions.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerActions.java
new file mode 100644
index 0000000..2f3a0cb
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerActions.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskmanager;
+
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+
+/**
+ * Interface for the communication of the {@link Task} with the {@link TaskManager}.
+ */
+public interface TaskManagerActions {
+
+ /**
+ * Notifies the task manager that the given task is in a final state.
+ *
+ * @param executionAttemptID Execution attempt ID of the task
+ */
+ void notifyFinalState(ExecutionAttemptID executionAttemptID);
+
+ /**
+ * Notifies the task manager about a fatal error occurred in the task.
+ *
+ * @param message Message to report
+ * @param cause Cause of the fatal error
+ */
+ void notifyFatalError(String message, Throwable cause);
+
+ /**
+ * Tells the task manager to fail the given task.
+ *
+ * @param executionAttemptID Execution attempt ID of the task to fail
+ * @param cause Cause of the failure
+ */
+ void failTask(ExecutionAttemptID executionAttemptID, Throwable cause);
+
+ /**
+ * Notifies the task manager about the task execution state update.
+ *
+ * @param taskExecutionState Task execution state update
+ */
+ void updateTaskExecutionState(TaskExecutionState taskExecutionState);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerConnection.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerConnection.java
deleted file mode 100644
index dc1b40f..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerConnection.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.taskmanager;
-
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
-
-/**
- * Interface for the communication of the {@link Task} with the {@link TaskManager}.
- */
-public interface TaskManagerConnection {
-
- /**
- * Notifies the task manager that the given task is in a final state.
- *
- * @param executionAttemptID Execution attempt ID of the task
- */
- void notifyFinalState(ExecutionAttemptID executionAttemptID);
-
- /**
- * Notifies the task manager about a fatal error occurred in the task.
- *
- * @param message Message to report
- * @param cause Cause of the fatal error
- */
- void notifyFatalError(String message, Throwable cause);
-
- /**
- * Tells the task manager to fail the given task.
- *
- * @param executionAttemptID Execution attempt ID of the task to fail
- * @param cause Cause of the failure
- */
- void failTask(ExecutionAttemptID executionAttemptID, Throwable cause);
-
- /**
- * Notifies the task manager about the task execution state update.
- *
- * @param taskExecutionState Task execution state update
- */
- void updateTaskExecutionState(TaskExecutionState taskExecutionState);
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 4007921..3ed8ea6 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -62,7 +62,8 @@ import org.apache.flink.runtime.metrics.util.MetricUtils
import org.apache.flink.runtime.process.ProcessReaper
import org.apache.flink.runtime.security.SecurityContext.{FlinkSecuredRunner, SecurityConfiguration}
import org.apache.flink.runtime.security.SecurityContext
-import org.apache.flink.runtime.taskexecutor.{TaskManagerConfiguration, TaskManagerServices, TaskManagerServicesConfiguration}
+import org.apache.flink.runtime.taskexecutor._
+import org.apache.flink.runtime.taskexecutor.utils.TaskExecutorMetricsInitializer
import org.apache.flink.runtime.util._
import org.apache.flink.runtime.{FlinkActor, LeaderSessionMessageFilter, LogMessages}
import org.apache.flink.util.NetUtils
@@ -144,7 +145,7 @@ class TaskManager(
protected val bcVarManager = new BroadcastVariableManager()
/** Handler for distributed files cached by this TaskManager */
- protected val fileCache = new FileCache(config.getConfiguration())
+ protected val fileCache = new FileCache(config.getTmpDirPaths())
private var taskManagerMetricGroup : TaskManagerMetricGroup = _
@@ -178,7 +179,7 @@ class TaskManager(
CheckpointResponder,
PartitionStateChecker,
ResultPartitionConsumableNotifier,
- TaskManagerConnection)] = None
+ TaskManagerActions)] = None
// --------------------------------------------------------------------------
// Actor messages and life cycle
@@ -915,9 +916,9 @@ class TaskManager(
val jobManagerGateway = new AkkaActorGateway(jobManager, leaderSessionID.orNull)
val taskManagerGateway = new AkkaActorGateway(self, leaderSessionID.orNull)
- val checkpointResponder = new ActorGatewayCheckpointResponder(jobManagerGateway);
+ val checkpointResponder = new ActorGatewayCheckpointResponder(jobManagerGateway)
- val taskManagerConnection = new ActorGatewayTaskManagerConnection(taskManagerGateway)
+ val taskManagerConnection = new ActorGatewayTaskManagerActions(taskManagerGateway)
val partitionStateChecker = new ActorGatewayPartitionStateChecker(
jobManagerGateway,
@@ -1971,6 +1972,5 @@ object TaskManager {
// --------------------------------------------------------------------------
/**
- }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java
index c369674..4db0d93 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java
@@ -21,7 +21,6 @@ package org.apache.flink.runtime.filecache;
import java.io.File;
import java.util.concurrent.Future;
-import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
import org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry;
import org.apache.flink.api.common.JobID;
@@ -62,8 +61,9 @@ public class FileCacheDeleteValidationTest {
@Before
public void setup() {
+ String[] tmpDirectories = System.getProperty("java.io.tmpdir").split(",|" + File.pathSeparator);
try {
- fileCache = new FileCache(new Configuration());
+ fileCache = new FileCache(tmpDirectories);
}
catch (Exception e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
index f8a0b6a..30dfef5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
@@ -42,7 +42,6 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index 7710fa9..f5fe52c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -20,8 +20,11 @@ package org.apache.flink.runtime.taskexecutor;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.clusterframework.types.SlotID;
+import org.apache.flink.runtime.filecache.FileCache;
import org.apache.flink.runtime.highavailability.NonHaServices;
import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
@@ -29,6 +32,7 @@ import org.apache.flink.runtime.io.network.NetworkEnvironment;
import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
import org.apache.flink.runtime.memory.MemoryManager;
import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRegistered;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRejected;
@@ -38,7 +42,6 @@ import org.apache.flink.runtime.rpc.TestingSerialRpcService;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.util.TestLogger;
-import org.hamcrest.Matchers;
import org.junit.Test;
import org.powermock.api.mockito.PowerMockito;
@@ -60,10 +63,14 @@ public class TaskExecutorTest extends TestLogger {
ResourceManagerGateway rmGateway = mock(ResourceManagerGateway.class);
TaskManagerConfiguration taskManagerServicesConfiguration = mock(TaskManagerConfiguration.class);
PowerMockito.when(taskManagerServicesConfiguration.getNumberSlots()).thenReturn(1);
+ PowerMockito.when(taskManagerServicesConfiguration.getConfiguration()).thenReturn(new Configuration());
+ PowerMockito.when(taskManagerServicesConfiguration.getTmpDirPaths()).thenReturn(new String[1]);
+
rpc.registerGateway(resourceManagerAddress, rmGateway);
TaskManagerLocation taskManagerLocation = mock(TaskManagerLocation.class);
when(taskManagerLocation.getResourceID()).thenReturn(resourceID);
+ when(taskManagerLocation.getHostname()).thenReturn("foobar");
NonHaServices haServices = new NonHaServices(resourceManagerAddress);
@@ -76,6 +83,9 @@ public class TaskExecutorTest extends TestLogger {
mock(NetworkEnvironment.class),
haServices,
mock(MetricRegistry.class),
+ mock(TaskManagerMetricGroup.class),
+ mock(BroadcastVariableManager.class),
+ mock(FileCache.class),
mock(FatalErrorHandler.class));
taskManager.start();
@@ -113,9 +123,12 @@ public class TaskExecutorTest extends TestLogger {
TaskManagerConfiguration taskManagerServicesConfiguration = mock(TaskManagerConfiguration.class);
PowerMockito.when(taskManagerServicesConfiguration.getNumberSlots()).thenReturn(1);
+ PowerMockito.when(taskManagerServicesConfiguration.getConfiguration()).thenReturn(new Configuration());
+ PowerMockito.when(taskManagerServicesConfiguration.getTmpDirPaths()).thenReturn(new String[1]);
TaskManagerLocation taskManagerLocation = mock(TaskManagerLocation.class);
when(taskManagerLocation.getResourceID()).thenReturn(resourceID);
+ when(taskManagerLocation.getHostname()).thenReturn("foobar");
TaskExecutor taskManager = new TaskExecutor(
taskManagerServicesConfiguration,
@@ -126,6 +139,9 @@ public class TaskExecutorTest extends TestLogger {
mock(NetworkEnvironment.class),
haServices,
mock(MetricRegistry.class),
+ mock(TaskManagerMetricGroup.class),
+ mock(BroadcastVariableManager.class),
+ mock(FileCache.class),
mock(FatalErrorHandler.class));
taskManager.start();
@@ -182,9 +198,12 @@ public class TaskExecutorTest extends TestLogger {
TaskManagerConfiguration taskManagerServicesConfiguration = mock(TaskManagerConfiguration.class);
PowerMockito.when(taskManagerServicesConfiguration.getNumberSlots()).thenReturn(1);
+ PowerMockito.when(taskManagerServicesConfiguration.getConfiguration()).thenReturn(new Configuration());
+ PowerMockito.when(taskManagerServicesConfiguration.getTmpDirPaths()).thenReturn(new String[1]);
TaskManagerLocation taskManagerLocation = mock(TaskManagerLocation.class);
when(taskManagerLocation.getResourceID()).thenReturn(resourceID);
+ when(taskManagerLocation.getHostname()).thenReturn("foobar");
TaskExecutor taskManager = new TaskExecutor(
taskManagerServicesConfiguration,
@@ -195,6 +214,9 @@ public class TaskExecutorTest extends TestLogger {
mock(NetworkEnvironment.class),
haServices,
mock(MetricRegistry.class),
+ mock(TaskManagerMetricGroup.class),
+ mock(BroadcastVariableManager.class),
+ mock(FileCache.class),
mock(FatalErrorHandler.class));
taskManager.start();
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
index ed107c7..d994134 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
@@ -169,7 +169,7 @@ public class TaskAsyncCallTest {
mock(IOManager.class),
networkEnvironment,
mock(BroadcastVariableManager.class),
- mock(TaskManagerConnection.class),
+ mock(TaskManagerActions.class),
mock(InputSplitProvider.class),
mock(CheckpointResponder.class),
libCache,
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskInputSplitProviderTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskInputSplitProviderTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskInputSplitProviderTest.java
index 642300d..777633d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskInputSplitProviderTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskInputSplitProviderTest.java
@@ -24,6 +24,7 @@ import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.instance.BaseTestingActorGateway;
import org.apache.flink.runtime.instance.ActorGateway;
import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.tasks.InputSplitProviderException;
import org.apache.flink.runtime.messages.JobManagerMessages;
import org.apache.flink.runtime.testingUtils.TestingUtils;
import org.junit.Test;
@@ -36,7 +37,7 @@ import java.util.concurrent.TimeUnit;
public class TaskInputSplitProviderTest {
@Test
- public void testRequestNextInputSplitWithInvalidExecutionID() {
+ public void testRequestNextInputSplitWithInvalidExecutionID() throws InputSplitProviderException {
final JobID jobID = new JobID();
final JobVertexID vertexID = new JobVertexID();
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
index 754361d..a2bfad5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
@@ -20,6 +20,7 @@ package org.apache.flink.runtime.taskmanager;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.TaskInfo;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.io.network.netty.PartitionStateChecker;
import org.apache.flink.runtime.io.network.partition.ResultPartitionConsumableNotifier;
import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
@@ -68,6 +69,7 @@ public class TaskStopTest {
when(tddMock.getTaskConfiguration()).thenReturn(mock(Configuration.class));
when(tddMock.getSerializedExecutionConfig()).thenReturn(mock(SerializedValue.class));
when(tddMock.getInvokableClassName()).thenReturn("className");
+ when(tddMock.getAllocationID()).thenReturn(mock(AllocationID.class));
task = new Task(
tddMock,
@@ -75,7 +77,7 @@ public class TaskStopTest {
mock(IOManager.class),
mock(NetworkEnvironment.class),
mock(BroadcastVariableManager.class),
- mock(TaskManagerConnection.class),
+ mock(TaskManagerActions.class),
mock(InputSplitProvider.class),
mock(CheckpointResponder.class),
mock(LibraryCacheManager.class),
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
index c38d23a..f28e58a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
@@ -101,7 +101,7 @@ public class TaskTest extends TestLogger {
private ActorGateway listenerGateway;
private ActorGatewayTaskExecutionStateListener listener;
- private ActorGatewayTaskManagerConnection taskManagerConnection;
+ private ActorGatewayTaskManagerActions taskManagerConnection;
private BlockingQueue<Object> taskManagerMessages;
private BlockingQueue<Object> jobManagerMessages;
@@ -117,7 +117,7 @@ public class TaskTest extends TestLogger {
listenerGateway = new ForwardingActorGateway(listenerMessages);
listener = new ActorGatewayTaskExecutionStateListener(listenerGateway);
- taskManagerConnection = new ActorGatewayTaskManagerConnection(taskManagerGateway);
+ taskManagerConnection = new ActorGatewayTaskManagerActions(taskManagerGateway);
awaitLatch = new OneShotLatch();
triggerLatch = new OneShotLatch();
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java
index 343affe..c067ca7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/source/InputFormatSourceFunction.java
@@ -26,6 +26,7 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.io.InputSplit;
import org.apache.flink.metrics.Counter;
import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
+import org.apache.flink.runtime.jobgraph.tasks.InputSplitProviderException;
import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
import java.util.Iterator;
@@ -146,7 +147,12 @@ public class InputFormatSourceFunction<OUT> extends RichParallelSourceFunction<O
return true;
}
- InputSplit split = provider.getNextInputSplit(getRuntimeContext().getUserCodeClassLoader());
+ final InputSplit split;
+ try {
+ split = provider.getNextInputSplit(getRuntimeContext().getUserCodeClassLoader());
+ } catch (InputSplitProviderException e) {
+ throw new RuntimeException("Could not retrieve next input split.", e);
+ }
if (split != null) {
this.nextSplit = split;
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java
index ee5a203..861665f 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java
@@ -48,7 +48,7 @@ import org.apache.flink.runtime.state.StreamStateHandle;
import org.apache.flink.runtime.state.TaskStateHandles;
import org.apache.flink.runtime.taskmanager.CheckpointResponder;
import org.apache.flink.runtime.taskmanager.Task;
-import org.apache.flink.runtime.taskmanager.TaskManagerConnection;
+import org.apache.flink.runtime.taskmanager.TaskManagerActions;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
import org.apache.flink.runtime.util.EnvironmentInformation;
import org.apache.flink.streaming.api.TimeCharacteristic;
@@ -60,6 +60,7 @@ import org.apache.flink.util.SerializedValue;
import org.junit.Test;
import java.io.EOFException;
+import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.net.URL;
@@ -161,17 +162,19 @@ public class InterruptSensitiveRestoreTest {
when(networkEnvironment.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class)))
.thenReturn(mock(TaskKvStateRegistry.class));
+ String[] tmpDirectories = EnvironmentInformation.getTemporaryFileDirectory().split(",|" + File.pathSeparator);
+
return new Task(
tdd,
mock(MemoryManager.class),
mock(IOManager.class),
networkEnvironment,
mock(BroadcastVariableManager.class),
- mock(TaskManagerConnection.class),
+ mock(TaskManagerActions.class),
mock(InputSplitProvider.class),
mock(CheckpointResponder.class),
new FallbackLibraryCacheManager(),
- new FileCache(new Configuration()),
+ new FileCache(tmpDirectories),
new TaskManagerRuntimeInfo(
"localhost", new Configuration(), EnvironmentInformation.getTemporaryFileDirectory()),
new UnregisteredTaskMetricsGroup(),
@@ -274,4 +277,4 @@ public class InterruptSensitiveRestoreTest {
fail("should never be called");
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
index 94f6d5a..ee0839f 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
@@ -51,7 +51,7 @@ import org.apache.flink.runtime.taskmanager.CheckpointResponder;
import org.apache.flink.runtime.taskmanager.Task;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import org.apache.flink.runtime.taskmanager.TaskExecutionStateListener;
-import org.apache.flink.runtime.taskmanager.TaskManagerConnection;
+import org.apache.flink.runtime.taskmanager.TaskManagerActions;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
@@ -238,7 +238,7 @@ public class StreamTaskTest {
mock(IOManager.class),
network,
mock(BroadcastVariableManager.class),
- mock(TaskManagerConnection.class),
+ mock(TaskManagerActions.class),
mock(InputSplitProvider.class),
mock(CheckpointResponder.class),
libCache,
[32/50] [abbrv] flink git commit: [FLINK-4689] [cluster management]
Implement a simple slot provider for the new job manager
Posted by tr...@apache.org.
[FLINK-4689] [cluster management] Implement a simple slot provider for the new job manager
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/0a02232e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/0a02232e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/0a02232e
Branch: refs/heads/flip-6
Commit: 0a02232ed69c7df210508bf944e39a0d61d059c9
Parents: cd069f0
Author: Kurt Young <yk...@gmail.com>
Authored: Sun Oct 16 22:20:38 2016 +0800
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:32 2016 +0100
----------------------------------------------------------------------
.../apache/flink/runtime/instance/SlotPool.java | 1 -
.../jobmanager/slots/PooledSlotProvider.java | 73 ++++++++++++++++++++
.../flink/runtime/jobmaster/JobMaster.java | 24 ++++---
3 files changed, 89 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/0a02232e/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
index e7857c1..de952c3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
@@ -135,7 +135,6 @@ public class SlotPool implements SlotOwner {
internalAllocateSlot(jobID, allocationID, resourceProfile, future);
- final SlotOwner owner = this;
return future.thenApplyAsync(
new ApplyFunction<SlotDescriptor, SimpleSlot>() {
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/0a02232e/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/PooledSlotProvider.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/PooledSlotProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/PooledSlotProvider.java
new file mode 100644
index 0000000..5655fc2
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/PooledSlotProvider.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmanager.slots;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.instance.SimpleSlot;
+import org.apache.flink.runtime.instance.SlotPool;
+import org.apache.flink.runtime.instance.SlotProvider;
+import org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException;
+import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * A simple pool based slot provider with {@link SlotPool} as the underlying storage.
+ */
+public class PooledSlotProvider implements SlotProvider {
+
+ /** The pool which holds all the slots. */
+ private final SlotPool slotPool;
+
+ /** The timeout for allocation. */
+ private final Time timeout;
+
+ public PooledSlotProvider(final SlotPool slotPool, final Time timeout) {
+ this.slotPool = slotPool;
+ this.timeout = timeout;
+ }
+
+ @Override
+ public Future<SimpleSlot> allocateSlot(ScheduledUnit task,
+ boolean allowQueued) throws NoResourceAvailableException
+ {
+ checkNotNull(task);
+
+ final JobID jobID = task.getTaskToExecute().getVertex().getJobId();
+ final Future<SimpleSlot> future = slotPool.allocateSimpleSlot(jobID, ResourceProfile.UNKNOWN);
+ try {
+ final SimpleSlot slot = future.get(timeout.getSize(), timeout.getUnit());
+ return FlinkCompletableFuture.completed(slot);
+ } catch (InterruptedException e) {
+ throw new NoResourceAvailableException("Could not allocate a slot because it's interrupted.");
+ } catch (ExecutionException e) {
+ throw new NoResourceAvailableException("Could not allocate a slot because some error occurred " +
+ "during allocation, " + e.getMessage());
+ } catch (TimeoutException e) {
+ throw new NoResourceAvailableException("Could not allocate a slot within time limit: " + timeout);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0a02232e/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index a7be476..05c20d3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -49,6 +49,7 @@ import org.apache.flink.runtime.executiongraph.restart.RestartStrategy;
import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.instance.Slot;
+import org.apache.flink.runtime.instance.SlotPool;
import org.apache.flink.runtime.io.network.PartitionState;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
@@ -56,7 +57,7 @@ import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobmanager.OnCompletionActions;
-import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
+import org.apache.flink.runtime.jobmanager.slots.PooledSlotProvider;
import org.apache.flink.runtime.jobmaster.message.ClassloadingProps;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
@@ -84,7 +85,6 @@ import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import org.apache.flink.runtime.util.SerializedThrowable;
import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.SerializedValue;
-
import org.slf4j.Logger;
import javax.annotation.Nullable;
@@ -93,6 +93,7 @@ import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -145,6 +146,9 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
/** The execution graph of this job */
private final ExecutionGraph executionGraph;
+ private final SlotPool slotPool;
+
+ private final Time allocationTimeout;
private volatile UUID leaderSessionID;
@@ -156,8 +160,6 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
/** Connection with ResourceManager, null if not located address yet or we close it initiative */
private ResourceManagerConnection resourceManagerConnection;
- // TODO - we need to replace this with the slot pool
- private final Scheduler scheduler;
// ------------------------------------------------------------------------
@@ -239,8 +241,8 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
-1,
log);
- // TODO - temp fix
- this.scheduler = new Scheduler(executorService);
+ this.slotPool = new SlotPool(executorService);
+ this.allocationTimeout = Time.of(5, TimeUnit.SECONDS);
}
//----------------------------------------------------------------------------------------------
@@ -262,6 +264,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
if (LEADER_ID_UPDATER.compareAndSet(this, null, leaderSessionID)) {
super.start();
+ slotPool.setJobManagerLeaderId(leaderSessionID);
log.info("Starting JobManager for job {} ({})", jobGraph.getName(), jobGraph.getJobID());
getSelf().startJobExecution();
} else {
@@ -337,7 +340,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
@Override
public void run() {
try {
- executionGraph.scheduleForExecution(scheduler);
+ executionGraph.scheduleForExecution(new PooledSlotProvider(slotPool, allocationTimeout));
} catch (Throwable t) {
executionGraph.fail(t);
}
@@ -365,6 +368,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
((StartStoppable) getSelf()).stop();
leaderSessionID = null;
+ slotPool.setJobManagerLeaderId(null);
executionGraph.suspend(cause);
// disconnect from resource manager:
@@ -783,9 +787,12 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
// TODO - add tests for comment in https://github.com/apache/flink/pull/2565
// verify the response with current connection
if (resourceManagerConnection != null
- && resourceManagerConnection.getTargetLeaderId().equals(success.getResourceManagerLeaderId())) {
+ && resourceManagerConnection.getTargetLeaderId().equals(success.getResourceManagerLeaderId()))
+ {
log.info("JobManager successfully registered at ResourceManager, leader id: {}.",
success.getResourceManagerLeaderId());
+ slotPool.setResourceManager(success.getResourceManagerLeaderId(),
+ resourceManagerConnection.getTargetGateway());
}
}
});
@@ -796,6 +803,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
resourceManagerConnection.close();
resourceManagerConnection = null;
}
+ slotPool.disconnectResourceManager();
}
//----------------------------------------------------------------------------------------------
[13/50] [abbrv] flink git commit: [FLINK-4478] [flip-6] Add
HeartbeatManager
Posted by tr...@apache.org.
[FLINK-4478] [flip-6] Add HeartbeatManager
Add a heartbeat manager abstraction which can monitor heartbeat targets. Whenever
no heartbeat signal has been received for a heartbeat timeout interval, the
heartbeat manager will issue a heartbeat timeout notification.
Add resourceID to HeartbeatListener.reportPayload
Replace scala future by Flink's futures
Add unmonitoring test
This closes #2435.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/36c38966
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/36c38966
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/36c38966
Branch: refs/heads/flip-6
Commit: 36c38966948cb195e9f802116d2b94a6b0d792a0
Parents: c7e1d8f
Author: Till Rohrmann <tr...@apache.org>
Authored: Thu Aug 25 14:05:07 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:29 2016 +0100
----------------------------------------------------------------------
.../runtime/heartbeat/HeartbeatListener.java | 62 ++++
.../runtime/heartbeat/HeartbeatManager.java | 67 ++++
.../runtime/heartbeat/HeartbeatManagerImpl.java | 328 +++++++++++++++++++
.../heartbeat/HeartbeatManagerSenderImpl.java | 81 +++++
.../runtime/heartbeat/HeartbeatTarget.java | 50 +++
.../runtime/heartbeat/HeartbeatManagerTest.java | 315 ++++++++++++++++++
.../slotmanager/SlotProtocolTest.java | 4 -
7 files changed, 903 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/36c38966/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatListener.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatListener.java b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatListener.java
new file mode 100644
index 0000000..8c08251
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatListener.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.heartbeat;
+
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.Future;
+
+/**
+ * Interface for the interaction with the {@link HeartbeatManager}. The heartbeat listener is used
+ * for the following things:
+ * <p>
+ * <ul>
+ * <il>Notifications about heartbeat timeouts</il>
+ * <li>Payload reports of incoming heartbeats</li>
+ * <li>Retrieval of payloads for outgoing heartbeats</li>
+ * </ul>
+ * @param <I> Type of the incoming payload
+ * @param <O> Type of the outgoing payload
+ */
+public interface HeartbeatListener<I, O> {
+
+ /**
+ * Callback which is called if a heartbeat for the machine identified by the given resource
+ * ID times out.
+ *
+ * @param resourceID Resource ID of the machine whose heartbeat has timed out
+ */
+ void notifyHeartbeatTimeout(ResourceID resourceID);
+
+ /**
+ * Callback which is called whenever a heartbeat with an associated payload is received. The
+ * carried payload is given to this method.
+ *
+ * @param resourceID Resource ID identifying the sender of the payload
+ * @param payload Payload of the received heartbeat
+ */
+ void reportPayload(ResourceID resourceID, I payload);
+
+ /**
+ * Retrieves the payload value for the next heartbeat message. Since the operation can happen
+ * asynchronously, the result is returned wrapped in a future.
+ *
+ * @return Future containing the next payload for heartbeats
+ */
+ Future<O> retrievePayload();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/36c38966/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManager.java
new file mode 100644
index 0000000..12918ed
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManager.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.heartbeat;
+
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+
+/**
+ * A heartbeat manager has to be able to do the following things:
+ *
+ * <ul>
+ * <li>Monitor {@link HeartbeatTarget} and report heartbeat timeouts for this target</li>
+ * <li>Stop monitoring a {@link HeartbeatTarget}</li>
+ * </ul>
+ *
+ *
+ * @param <I> Type of the incoming payload
+ * @param <O> Type of the outgoing payload
+ */
+public interface HeartbeatManager<I, O> {
+
+ /**
+ * Start monitoring a {@link HeartbeatTarget}. Heartbeat timeouts for this target are reported
+ * to the {@link HeartbeatListener} associated with this heartbeat manager.
+ *
+ * @param resourceID Resource ID identifying the heartbeat target
+ * @param heartbeatTarget Interface to send heartbeat requests and responses to the heartbeat
+ * target
+ */
+ void monitorTarget(ResourceID resourceID, HeartbeatTarget<O> heartbeatTarget);
+
+ /**
+ * Stops monitoring the heartbeat target with the associated resource ID.
+ *
+ * @param resourceID Resource ID of the heartbeat target which shall no longer be monitored
+ */
+ void unmonitorTarget(ResourceID resourceID);
+
+ /**
+ * Starts the heartbeat manager with the given {@link HeartbeatListener}. The heartbeat listener
+ * is notified about heartbeat timeouts and heartbeat payloads are reported and retrieved to
+ * and from it.
+ *
+ * @param heartbeatListener Heartbeat listener associated with the heartbeat manager
+ */
+ void start(HeartbeatListener<I, O> heartbeatListener);
+
+ /**
+ * Stops the heartbeat manager.
+ */
+ void stop();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/36c38966/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerImpl.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerImpl.java
new file mode 100644
index 0000000..042f95b
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerImpl.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.heartbeat;
+
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.AcceptFunction;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+
+import javax.annotation.concurrent.ThreadSafe;
+import java.util.Collection;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Heartbeat manager implementation. The heartbeat manager maintains a map of heartbeat monitors
+ * and resource IDs. Each monitor will be updated when a new heartbeat of the associated machine has
+ * been received. If the monitor detects that a heartbeat has timed out, it will notify the
+ * {@link HeartbeatListener} about it. A heartbeat times out iff no heartbeat signal has been
+ * received within a given timeout interval.
+ *
+ * @param <I> Type of the incoming heartbeat payload
+ * @param <O> Type of the outgoing heartbeat payload
+ */
+@ThreadSafe
+public class HeartbeatManagerImpl<I, O> implements HeartbeatManager<I, O>, HeartbeatTarget<I> {
+
+ /** Heartbeat timeout interval in milli seconds */
+ private final long heartbeatTimeoutIntervalMs;
+
+ /** Resource ID which is used to mark one own's heartbeat signals */
+ private final ResourceID ownResourceID;
+
+ /** Executor service used to run heartbeat timeout notifications */
+ private final ScheduledExecutorService scheduledExecutorService;
+
+ protected final Logger log;
+
+ /** Map containing the heartbeat monitors associated with the respective resource ID */
+ private final ConcurrentHashMap<ResourceID, HeartbeatManagerImpl.HeartbeatMonitor<O>> heartbeatTargets;
+
+ /** Execution context used to run future callbacks */
+ private final Executor executor;
+
+ /** Heartbeat listener with which the heartbeat manager has been associated */
+ private HeartbeatListener<I, O> heartbeatListener;
+
+ /** Running state of the heartbeat manager */
+ protected volatile boolean stopped;
+
+ public HeartbeatManagerImpl(
+ long heartbeatTimeoutIntervalMs,
+ ResourceID ownResourceID,
+ Executor executor,
+ ScheduledExecutorService scheduledExecutorService,
+ Logger log) {
+ Preconditions.checkArgument(heartbeatTimeoutIntervalMs > 0L, "The heartbeat timeout has to be larger than 0.");
+
+ this.heartbeatTimeoutIntervalMs = heartbeatTimeoutIntervalMs;
+ this.ownResourceID = Preconditions.checkNotNull(ownResourceID);
+ this.scheduledExecutorService = Preconditions.checkNotNull(scheduledExecutorService);
+ this.log = Preconditions.checkNotNull(log);
+ this.executor = Preconditions.checkNotNull(executor);
+ this.heartbeatTargets = new ConcurrentHashMap<>(16);
+
+ stopped = true;
+ }
+
+ //----------------------------------------------------------------------------------------------
+ // Getters
+ //----------------------------------------------------------------------------------------------
+
+ ResourceID getOwnResourceID() {
+ return ownResourceID;
+ }
+
+ Executor getExecutor() {
+ return executor;
+ }
+
+ HeartbeatListener<I, O> getHeartbeatListener() {
+ return heartbeatListener;
+ }
+
+ Collection<HeartbeatManagerImpl.HeartbeatMonitor<O>> getHeartbeatTargets() {
+ return heartbeatTargets.values();
+ }
+
+ //----------------------------------------------------------------------------------------------
+ // HeartbeatManager methods
+ //----------------------------------------------------------------------------------------------
+
+ @Override
+ public void monitorTarget(ResourceID resourceID, HeartbeatTarget<O> heartbeatTarget) {
+ if (!stopped) {
+ if (heartbeatTargets.containsKey(resourceID)) {
+ log.info("The target with resource ID {} is already been monitored.", resourceID);
+ } else {
+ HeartbeatManagerImpl.HeartbeatMonitor<O> heartbeatMonitor = new HeartbeatManagerImpl.HeartbeatMonitor<>(
+ resourceID,
+ heartbeatTarget,
+ scheduledExecutorService,
+ heartbeatListener,
+ heartbeatTimeoutIntervalMs);
+
+ heartbeatTargets.put(
+ resourceID,
+ heartbeatMonitor);
+
+ // check if we have stopped in the meantime (concurrent stop operation)
+ if (stopped) {
+ heartbeatMonitor.cancel();
+
+ heartbeatTargets.remove(resourceID);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void unmonitorTarget(ResourceID resourceID) {
+ if (!stopped) {
+ HeartbeatManagerImpl.HeartbeatMonitor<O> heartbeatMonitor = heartbeatTargets.remove(resourceID);
+
+ if (heartbeatMonitor != null) {
+ heartbeatMonitor.cancel();
+ }
+ }
+ }
+
+ @Override
+ public void start(HeartbeatListener<I, O> heartbeatListener) {
+ Preconditions.checkState(stopped, "Cannot start an already started heartbeat manager.");
+
+ stopped = false;
+
+ this.heartbeatListener = Preconditions.checkNotNull(heartbeatListener);
+ }
+
+ @Override
+ public void stop() {
+ stopped = true;
+
+ for (HeartbeatManagerImpl.HeartbeatMonitor<O> heartbeatMonitor : heartbeatTargets.values()) {
+ heartbeatMonitor.cancel();
+ }
+
+ heartbeatTargets.clear();
+ }
+
+ //----------------------------------------------------------------------------------------------
+ // HeartbeatTarget methods
+ //----------------------------------------------------------------------------------------------
+
+ @Override
+ public void sendHeartbeat(ResourceID resourceID, I payload) {
+ if (!stopped) {
+ log.debug("Received heartbeat from {}.", resourceID);
+ reportHeartbeat(resourceID);
+
+ if (payload != null) {
+ heartbeatListener.reportPayload(resourceID, payload);
+ }
+ }
+ }
+
+ @Override
+ public void requestHeartbeat(ResourceID resourceID, I payload) {
+ if (!stopped) {
+ log.debug("Received heartbeat request from {}.", resourceID);
+
+ final HeartbeatTarget<O> heartbeatTarget = reportHeartbeat(resourceID);
+
+ if (heartbeatTarget != null) {
+ if (payload != null) {
+ heartbeatListener.reportPayload(resourceID, payload);
+ }
+
+ Future<O> futurePayload = heartbeatListener.retrievePayload();
+
+ if (futurePayload != null) {
+ futurePayload.thenAcceptAsync(new AcceptFunction<O>() {
+ @Override
+ public void accept(O retrievedPayload) {
+ heartbeatTarget.sendHeartbeat(getOwnResourceID(), retrievedPayload);
+ }
+ }, executor);
+ } else {
+ heartbeatTarget.sendHeartbeat(ownResourceID, null);
+ }
+ }
+ }
+ }
+
+ HeartbeatTarget<O> reportHeartbeat(ResourceID resourceID) {
+ if (heartbeatTargets.containsKey(resourceID)) {
+ HeartbeatManagerImpl.HeartbeatMonitor<O> heartbeatMonitor = heartbeatTargets.get(resourceID);
+ heartbeatMonitor.reportHeartbeat();
+
+ return heartbeatMonitor.getHeartbeatTarget();
+ } else {
+ return null;
+ }
+ }
+
+ //----------------------------------------------------------------------------------------------
+ // Utility classes
+ //----------------------------------------------------------------------------------------------
+
+ /**
+ * Heartbeat monitor which manages the heartbeat state of the associated heartbeat target. The
+ * monitor notifies the {@link HeartbeatListener} whenever it has not seen a heartbeat signal
+ * in the specified heartbeat timeout interval. Each heartbeat signal resets this timer.
+ *
+ * @param <O> Type of the payload being sent to the associated heartbeat target
+ */
+ static class HeartbeatMonitor<O> implements Runnable {
+
+ /** Resource ID of the monitored heartbeat target */
+ private final ResourceID resourceID;
+
+ /** Associated heartbeat target */
+ private final HeartbeatTarget<O> heartbeatTarget;
+
+ private final ScheduledExecutorService scheduledExecutorService;
+
+ /** Listener which is notified about heartbeat timeouts */
+ private final HeartbeatListener<?, ?> heartbeatListener;
+
+ /** Maximum heartbeat timeout interval */
+ private final long heartbeatTimeoutIntervalMs;
+
+ private volatile ScheduledFuture<?> futureTimeout;
+
+ private final AtomicReference<State> state = new AtomicReference<>(State.RUNNING);
+
+ HeartbeatMonitor(
+ ResourceID resourceID,
+ HeartbeatTarget<O> heartbeatTarget,
+ ScheduledExecutorService scheduledExecutorService,
+ HeartbeatListener<?, O> heartbeatListener,
+ long heartbeatTimeoutIntervalMs) {
+
+ this.resourceID = Preconditions.checkNotNull(resourceID);
+ this.heartbeatTarget = Preconditions.checkNotNull(heartbeatTarget);
+ this.scheduledExecutorService = Preconditions.checkNotNull(scheduledExecutorService);
+ this.heartbeatListener = Preconditions.checkNotNull(heartbeatListener);
+
+ Preconditions.checkArgument(heartbeatTimeoutIntervalMs >= 0L, "The heartbeat timeout interval has to be larger than 0.");
+ this.heartbeatTimeoutIntervalMs = heartbeatTimeoutIntervalMs;
+
+ resetHeartbeatTimeout(heartbeatTimeoutIntervalMs);
+ }
+
+ HeartbeatTarget<O> getHeartbeatTarget() {
+ return heartbeatTarget;
+ }
+
+ void reportHeartbeat() {
+ resetHeartbeatTimeout(heartbeatTimeoutIntervalMs);
+ }
+
+ void resetHeartbeatTimeout(long heartbeatTimeout) {
+ if (state.get() == State.RUNNING) {
+ cancelTimeout();
+
+ futureTimeout = scheduledExecutorService.schedule(this, heartbeatTimeout, TimeUnit.MILLISECONDS);
+
+ // Double check for concurrent accesses (e.g. a firing of the scheduled future)
+ if (state.get() != State.RUNNING) {
+ cancelTimeout();
+ }
+ }
+ }
+
+ void cancel() {
+ // we can only cancel if we are in state running
+ if (state.compareAndSet(State.RUNNING, State.CANCELED)) {
+ cancelTimeout();
+ }
+ }
+
+ private void cancelTimeout() {
+ if (futureTimeout != null) {
+ futureTimeout.cancel(true);
+ }
+ }
+
+ public boolean isCanceled() {
+ return state.get() == State.CANCELED;
+ }
+
+ @Override
+ public void run() {
+ // The heartbeat has timed out if we're in state running
+ if (state.compareAndSet(State.RUNNING, State.TIMEOUT)) {
+ heartbeatListener.notifyHeartbeatTimeout(resourceID);
+ }
+ }
+
+ private enum State {
+ RUNNING,
+ TIMEOUT,
+ CANCELED
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/36c38966/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerSenderImpl.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerSenderImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerSenderImpl.java
new file mode 100644
index 0000000..588ba7f
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerSenderImpl.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.heartbeat;
+
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.AcceptFunction;
+import org.apache.flink.runtime.concurrent.Future;
+import org.slf4j.Logger;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * {@link HeartbeatManager} implementation which regularly requests a heartbeat response from
+ * its monitored {@link HeartbeatTarget}. The heartbeat period is configurable.
+ *
+ * @param <I> Type of the incoming heartbeat payload
+ * @param <O> Type of the outgoind heartbeat payload
+ */
+public class HeartbeatManagerSenderImpl<I, O> extends HeartbeatManagerImpl<I, O> implements Runnable {
+
+ private final ScheduledFuture<?> triggerFuture;
+
+ public HeartbeatManagerSenderImpl(
+ long heartbeatPeriod,
+ long heartbeatTimeout,
+ ResourceID ownResourceID,
+ ExecutorService executorService,
+ ScheduledExecutorService scheduledExecutorService,
+ Logger log) {
+ super(heartbeatTimeout, ownResourceID, executorService, scheduledExecutorService, log);
+
+ triggerFuture = scheduledExecutorService.scheduleAtFixedRate(this, 0L, heartbeatPeriod, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public void run() {
+ if (!stopped) {
+ log.debug("Trigger heartbeat request.");
+ for (HeartbeatMonitor<O> heartbeatMonitor : getHeartbeatTargets()) {
+ Future<O> futurePayload = getHeartbeatListener().retrievePayload();
+ final HeartbeatTarget<O> heartbeatTarget = heartbeatMonitor.getHeartbeatTarget();
+
+ if (futurePayload != null) {
+ futurePayload.thenAcceptAsync(new AcceptFunction<O>() {
+ @Override
+ public void accept(O payload) {
+ heartbeatTarget.requestHeartbeat(getOwnResourceID(), payload);
+ }
+ }, getExecutor());
+ } else {
+ heartbeatTarget.requestHeartbeat(getOwnResourceID(), null);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void stop() {
+ triggerFuture.cancel(true);
+ super.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/36c38966/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatTarget.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatTarget.java b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatTarget.java
new file mode 100644
index 0000000..ef953de
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatTarget.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.heartbeat;
+
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+
+/**
+ * Interface for components which can be sent heartbeats and from which one can request a
+ * heartbeat response. Both the heartbeat response as well as the heartbeat request can carry a
+ * payload. This payload is reported to the heartbeat target and contains additional information.
+ * The payload can be empty which is indicated by a null value.
+ *
+ * @param <I> Type of the payload which is sent to the heartbeat target
+ */
+public interface HeartbeatTarget<I> {
+
+ /**
+ * Sends a heartbeat response to the target. Each heartbeat response can carry a payload which
+ * contains additional information for the heartbeat target.
+ *
+ * @param resourceID Resource ID identifying the machine for which a heartbeat shall be reported.
+ * @param payload Payload of the heartbeat response. Null indicates an empty payload.
+ */
+ void sendHeartbeat(ResourceID resourceID, I payload);
+
+ /**
+ * Requests a heartbeat from the target. Each heartbeat request can carry a payload which
+ * contains additional information for the heartbeat target.
+ *
+ * @param resourceID Resource ID identifying the machine issuing the heartbeat request.
+ * @param payload Payload of the heartbeat response. Null indicates an empty payload.
+ */
+ void requestHeartbeat(ResourceID resourceID, I payload);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/36c38966/flink-runtime/src/test/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerTest.java
new file mode 100644
index 0000000..1c62f17
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerTest.java
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.heartbeat;
+
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.CompletableFuture;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.util.DirectExecutorService;
+import org.apache.flink.util.TestLogger;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.atLeast;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+
+public class HeartbeatManagerTest extends TestLogger {
+ private static final Logger LOG = LoggerFactory.getLogger(HeartbeatManagerTest.class);
+
+ /**
+ * Tests that regular heartbeat signal triggers the right callback functions in the
+ * {@link HeartbeatListener}.
+ */
+ @Test
+ public void testRegularHeartbeat() {
+ long heartbeatTimeout = 1000L;
+ ResourceID ownResourceID = new ResourceID("foobar");
+ ResourceID targetResourceID = new ResourceID("barfoo");
+ HeartbeatListener<Object, Object> heartbeatListener = mock(HeartbeatListener.class);
+ ScheduledExecutorService scheduledExecutorService = mock(ScheduledExecutorService.class);
+
+ Object expectedObject = new Object();
+
+ when(heartbeatListener.retrievePayload()).thenReturn(FlinkCompletableFuture.completed(expectedObject));
+
+ HeartbeatManagerImpl<Object, Object> heartbeatManager = new HeartbeatManagerImpl<>(
+ heartbeatTimeout,
+ ownResourceID,
+ new DirectExecutorService(),
+ scheduledExecutorService,
+ LOG);
+
+ heartbeatManager.start(heartbeatListener);
+
+ HeartbeatTarget<Object> heartbeatTarget = mock(HeartbeatTarget.class);
+
+ heartbeatManager.monitorTarget(targetResourceID, heartbeatTarget);
+
+ heartbeatManager.requestHeartbeat(targetResourceID, expectedObject);
+
+ verify(heartbeatListener, times(1)).reportPayload(targetResourceID, expectedObject);
+ verify(heartbeatListener, times(1)).retrievePayload();
+ verify(heartbeatTarget, times(1)).sendHeartbeat(ownResourceID, expectedObject);
+
+ heartbeatManager.sendHeartbeat(targetResourceID, expectedObject);
+
+ verify(heartbeatListener, times(2)).reportPayload(targetResourceID, expectedObject);
+ }
+
+ /**
+ * Tests that the heartbeat monitors are updated when receiving a new heartbeat signal.
+ */
+ @Test
+ public void testHeartbeatMonitorUpdate() {
+ long heartbeatTimeout = 1000L;
+ ResourceID ownResourceID = new ResourceID("foobar");
+ ResourceID targetResourceID = new ResourceID("barfoo");
+ HeartbeatListener<Object, Object> heartbeatListener = mock(HeartbeatListener.class);
+ ScheduledExecutorService scheduledExecutorService = mock(ScheduledExecutorService.class);
+ ScheduledFuture<?> scheduledFuture = mock(ScheduledFuture.class);
+
+ doReturn(scheduledFuture).when(scheduledExecutorService).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class));
+
+ Object expectedObject = new Object();
+
+ when(heartbeatListener.retrievePayload()).thenReturn(FlinkCompletableFuture.completed(expectedObject));
+
+ HeartbeatManagerImpl<Object, Object> heartbeatManager = new HeartbeatManagerImpl<>(
+ heartbeatTimeout,
+ ownResourceID,
+ new DirectExecutorService(),
+ scheduledExecutorService,
+ LOG);
+
+ heartbeatManager.start(heartbeatListener);
+
+ HeartbeatTarget<Object> heartbeatTarget = mock(HeartbeatTarget.class);
+
+ heartbeatManager.monitorTarget(targetResourceID, heartbeatTarget);
+
+ heartbeatManager.sendHeartbeat(targetResourceID, expectedObject);
+
+ verify(scheduledFuture, times(1)).cancel(true);
+ verify(scheduledExecutorService, times(2)).schedule(any(Runnable.class), eq(heartbeatTimeout), eq(TimeUnit.MILLISECONDS));
+ }
+
+ /**
+ * Tests that a heartbeat timeout is signaled if the heartbeat is not reported in time.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testHeartbeatTimeout() throws Exception {
+ long heartbeatTimeout = 100L;
+ int numHeartbeats = 10;
+ long heartbeatInterval = 20L;
+ Object payload = new Object();
+
+ ResourceID ownResourceID = new ResourceID("foobar");
+ ResourceID targetResourceID = new ResourceID("barfoo");
+ TestingHeartbeatListener heartbeatListener = new TestingHeartbeatListener(payload);
+ ScheduledExecutorService scheduledExecutorService = mock(ScheduledExecutorService.class);
+ ScheduledFuture<?> scheduledFuture = mock(ScheduledFuture.class);
+
+ doReturn(scheduledFuture).when(scheduledExecutorService).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class));
+
+ Object expectedObject = new Object();
+
+ HeartbeatManagerImpl<Object, Object> heartbeatManager = new HeartbeatManagerImpl<>(
+ heartbeatTimeout,
+ ownResourceID,
+ new DirectExecutorService(),
+ new ScheduledThreadPoolExecutor(1),
+ LOG);
+
+ heartbeatManager.start(heartbeatListener);
+
+ HeartbeatTarget<Object> heartbeatTarget = mock(HeartbeatTarget.class);
+
+ Future<ResourceID> timeoutFuture = heartbeatListener.getTimeoutFuture();
+
+ heartbeatManager.monitorTarget(targetResourceID, heartbeatTarget);
+
+ for (int i = 0; i < numHeartbeats; i++) {
+ heartbeatManager.sendHeartbeat(targetResourceID, expectedObject);
+ Thread.sleep(heartbeatInterval);
+ }
+
+ assertFalse(timeoutFuture.isDone());
+
+ ResourceID timeoutResourceID = timeoutFuture.get(2 * heartbeatTimeout, TimeUnit.MILLISECONDS);
+
+ assertEquals(targetResourceID, timeoutResourceID);
+ }
+
+ /**
+ * Tests the heartbeat interplay between the {@link HeartbeatManagerImpl} and the
+ * {@link HeartbeatManagerSenderImpl}. The sender should regularly trigger heartbeat requests
+ * which are fulfilled by the receiver. Upon stopping the receiver, the sender should notify
+ * the heartbeat listener about the heartbeat timeout.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testHeartbeatCluster() throws Exception {
+ long heartbeatTimeout = 100L;
+ long heartbeatPeriod = 20L;
+ Object object = new Object();
+ Object object2 = new Object();
+ ResourceID resourceID = new ResourceID("foobar");
+ ResourceID resourceID2 = new ResourceID("barfoo");
+ HeartbeatListener<Object, Object> heartbeatListener = mock(HeartbeatListener.class);
+
+ when(heartbeatListener.retrievePayload()).thenReturn(FlinkCompletableFuture.completed(object));
+
+ TestingHeartbeatListener heartbeatListener2 = new TestingHeartbeatListener(object2);
+
+ Future<ResourceID> futureTimeout = heartbeatListener2.getTimeoutFuture();
+
+ HeartbeatManagerImpl<Object, Object> heartbeatManager = new HeartbeatManagerImpl<>(
+ heartbeatTimeout,
+ resourceID,
+ new DirectExecutorService(),
+ new ScheduledThreadPoolExecutor(1),
+ LOG);
+
+ HeartbeatManagerSenderImpl<Object, Object> heartbeatManager2 = new HeartbeatManagerSenderImpl<>(
+ heartbeatPeriod,
+ heartbeatTimeout,
+ resourceID2,
+ new DirectExecutorService(),
+ new ScheduledThreadPoolExecutor(1),
+ LOG);;
+
+ heartbeatManager.start(heartbeatListener);
+ heartbeatManager2.start(heartbeatListener2);
+
+ heartbeatManager.monitorTarget(resourceID2, heartbeatManager2);
+ heartbeatManager2.monitorTarget(resourceID, heartbeatManager);
+
+ Thread.sleep(2 * heartbeatTimeout);
+
+ assertFalse(futureTimeout.isDone());
+
+ heartbeatManager.stop();
+
+ ResourceID timeoutResourceID = futureTimeout.get(2 * heartbeatTimeout, TimeUnit.MILLISECONDS);
+
+ assertEquals(resourceID, timeoutResourceID);
+
+ int numberHeartbeats = (int) (2 * heartbeatTimeout / heartbeatPeriod);
+
+ verify(heartbeatListener, atLeast(numberHeartbeats / 2)).reportPayload(resourceID2, object2);
+ assertTrue(heartbeatListener2.getNumberHeartbeatReports() >= numberHeartbeats / 2);
+ }
+
+ /**
+ * Tests that after unmonitoring a target, there won't be a timeout triggered
+ */
+ @Test
+ public void testTargetUnmonitoring() throws InterruptedException, ExecutionException {
+ // this might be too aggresive for Travis, let's see...
+ long heartbeatTimeout = 100L;
+ ResourceID resourceID = new ResourceID("foobar");
+ ResourceID targetID = new ResourceID("target");
+ Object object = new Object();
+
+ HeartbeatManager<Object, Object> heartbeatManager = new HeartbeatManagerImpl<>(
+ heartbeatTimeout,
+ resourceID,
+ new DirectExecutorService(),
+ new ScheduledThreadPoolExecutor(1),
+ LOG);
+
+ TestingHeartbeatListener heartbeatListener = new TestingHeartbeatListener(object);
+
+ heartbeatManager.start(heartbeatListener);
+
+ heartbeatManager.monitorTarget(targetID, mock(HeartbeatTarget.class));
+
+ heartbeatManager.unmonitorTarget(targetID);
+
+ Future<ResourceID> timeout = heartbeatListener.getTimeoutFuture();
+
+
+ try {
+ timeout.get(2 * heartbeatTimeout, TimeUnit.MILLISECONDS);
+ fail("Timeout should time out.");
+ } catch (TimeoutException e) {
+ // the timeout should not be completed since we unmonitored the target
+ }
+ }
+
+ static class TestingHeartbeatListener implements HeartbeatListener<Object, Object> {
+
+ private final CompletableFuture<ResourceID> future = new FlinkCompletableFuture<>();
+
+ private final Object payload;
+
+ private int numberHeartbeatReports;
+
+ TestingHeartbeatListener(Object payload) {
+ this.payload = payload;
+ }
+
+ public Future<ResourceID> getTimeoutFuture() {
+ return future;
+ }
+
+ public int getNumberHeartbeatReports() {
+ return numberHeartbeatReports;
+ }
+
+ @Override
+ public void notifyHeartbeatTimeout(ResourceID resourceID) {
+ future.complete(resourceID);
+ }
+
+ @Override
+ public void reportPayload(ResourceID resourceID, Object payload) {
+ numberHeartbeatReports++;
+ }
+
+ @Override
+ public Future<Object> retrievePayload() {
+ return FlinkCompletableFuture.completed(payload);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/36c38966/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
index 805ea71..a87fe42 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
@@ -25,7 +25,6 @@ import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.clusterframework.types.SlotID;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
-import org.apache.flink.runtime.concurrent.impl.FlinkFuture;
import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
import org.apache.flink.runtime.jobmaster.JobMasterGateway;
import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
@@ -43,12 +42,9 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
import java.util.Collections;
import java.util.UUID;
-import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import static org.mockito.Matchers.any;
[25/50] [abbrv] flink git commit: [FLINK-4360] [tm] Implement TM ->
JM registration logic
Posted by tr...@apache.org.
[FLINK-4360] [tm] Implement TM -> JM registration logic
Upon requesting a slot for a new job, the TaskManager registers this job at the
JobLeaderService. The job leader service is responsible to monitor job leader changes
for all registered jobs. In case of a new job leader, the service will try to establish
a connection to the new job leader. Upon establishing the connection the task manager
is informed about it. The task manager will then offer all allocated but not yet active
slots to the new job leader.
Implement JobLeaderService
The JobLeaderService is responsible for establishing a connection to the JM leader of a given
job.
Disable TaskExecutorTest#testRejectAllocationRequestsForOutOfSyncSlots
Add simple task submission test
Add job leader detection test case
Add task slot acceptance test
Fix RpcCompletenessTest
Add comments
This closes #2640.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2b234521
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2b234521
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2b234521
Branch: refs/heads/flip-6
Commit: 2b234521388630274f63cdd7e52fbbe4cd7ee28b
Parents: a122bfd
Author: Till Rohrmann <tr...@apache.org>
Authored: Wed Oct 5 17:02:06 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:31 2016 +0100
----------------------------------------------------------------------
.../org/apache/flink/util/ReflectionUtil.java | 110 ++++
.../deployment/TaskDeploymentDescriptor.java | 11 +-
.../runtime/executiongraph/ExecutionVertex.java | 2 +
.../HighAvailabilityServices.java | 3 +-
.../runtime/highavailability/NonHaServices.java | 4 +-
.../highavailability/ZookeeperHaServices.java | 2 +-
.../jobmaster/JMTMRegistrationSuccess.java | 45 ++
.../flink/runtime/jobmaster/JobMaster.java | 19 +
.../runtime/jobmaster/JobMasterGateway.java | 36 ++
.../registration/RegisteredRpcConnection.java | 2 +-
.../resourcemanager/ResourceManager.java | 2 +-
.../slotmanager/SlotManager.java | 8 +-
.../runtime/taskexecutor/JobLeaderListener.java | 60 +++
.../runtime/taskexecutor/JobLeaderService.java | 390 ++++++++++++++
.../taskexecutor/JobManagerConnection.java | 23 +-
.../runtime/taskexecutor/JobManagerTable.java | 59 +++
.../runtime/taskexecutor/TaskExecutor.java | 522 ++++++++++++++-----
.../taskexecutor/TaskExecutorGateway.java | 25 +-
...TaskExecutorToResourceManagerConnection.java | 5 +
.../runtime/taskexecutor/TaskManagerRunner.java | 2 +
.../taskexecutor/TaskManagerServices.java | 24 +-
.../exceptions/SlotAllocationException.java | 39 ++
.../taskexecutor/slot/TaskSlotTable.java | 39 +-
.../apache/flink/runtime/taskmanager/Task.java | 2 +-
.../TaskDeploymentDescriptorTest.java | 6 +-
.../TestingHighAvailabilityServices.java | 2 +-
.../metrics/groups/TaskManagerGroupTest.java | 10 +-
.../slotmanager/SlotManagerTest.java | 2 +-
.../slotmanager/SlotProtocolTest.java | 8 +-
.../flink/runtime/rpc/RpcCompletenessTest.java | 29 +-
.../runtime/taskexecutor/TaskExecutorTest.java | 431 ++++++++++++++-
.../runtime/taskmanager/TaskAsyncCallTest.java | 3 +-
.../runtime/taskmanager/TaskManagerTest.java | 75 +--
.../flink/runtime/taskmanager/TaskStopTest.java | 2 +-
.../flink/runtime/taskmanager/TaskTest.java | 21 +-
.../tasks/InterruptSensitiveRestoreTest.java | 32 +-
.../streaming/runtime/tasks/StreamTaskTest.java | 25 +-
37 files changed, 1801 insertions(+), 279 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-core/src/main/java/org/apache/flink/util/ReflectionUtil.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/ReflectionUtil.java b/flink-core/src/main/java/org/apache/flink/util/ReflectionUtil.java
index b851eba..2883570 100644
--- a/flink-core/src/main/java/org/apache/flink/util/ReflectionUtil.java
+++ b/flink-core/src/main/java/org/apache/flink/util/ReflectionUtil.java
@@ -23,6 +23,9 @@ import org.apache.flink.annotation.Internal;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
@Internal
public final class ReflectionUtil {
@@ -151,6 +154,113 @@ public final class ReflectionUtil {
}
/**
+ * Extract the full template type information from the given type's template parameter at the
+ * given position.
+ *
+ * @param type type to extract the full template parameter information from
+ * @param templatePosition describing at which position the template type parameter is
+ * @return Full type information describing the template parameter's type
+ */
+ public static FullTypeInfo getFullTemplateType(Type type, int templatePosition) {
+ if (type instanceof ParameterizedType) {
+ return getFullTemplateType(((ParameterizedType) type).getActualTypeArguments()[templatePosition]);
+ } else {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ /**
+ * Extract the full type information from the given type.
+ *
+ * @param type to be analyzed
+ * @return Full type information describing the given type
+ */
+ public static FullTypeInfo getFullTemplateType(Type type) {
+ if (type instanceof ParameterizedType) {
+ ParameterizedType parameterizedType = (ParameterizedType) type;
+
+ FullTypeInfo[] templateTypeInfos = new FullTypeInfo[parameterizedType.getActualTypeArguments().length];
+
+ for (int i = 0; i < parameterizedType.getActualTypeArguments().length; i++) {
+ templateTypeInfos[i] = getFullTemplateType(parameterizedType.getActualTypeArguments()[i]);
+ }
+
+ return new FullTypeInfo((Class<?>)parameterizedType.getRawType(), templateTypeInfos);
+ } else {
+ return new FullTypeInfo((Class<?>) type, null);
+ }
+ }
+
+ /**
+ * Container for the full type information of a type. This means that it contains the
+ * {@link Class} object and for each template parameter it contains a full type information
+ * describing the type.
+ */
+ public static class FullTypeInfo {
+ private final Class<?> clazz;
+ private final FullTypeInfo[] templateTypeInfos;
+
+
+ public FullTypeInfo(Class<?> clazz, FullTypeInfo[] templateTypeInfos) {
+ this.clazz = Preconditions.checkNotNull(clazz);
+ this.templateTypeInfos = templateTypeInfos;
+ }
+
+ public Class<?> getClazz() {
+ return clazz;
+ }
+
+ public FullTypeInfo[] getTemplateTypeInfos() {
+ return templateTypeInfos;
+ }
+
+ public Iterator<Class<?>> getClazzIterator() {
+ UnionIterator<Class<?>> unionIterator = new UnionIterator<>();
+
+ unionIterator.add(Collections.<Class<?>>singleton(clazz).iterator());
+
+ if (templateTypeInfos != null) {
+ for (int i = 0; i < templateTypeInfos.length; i++) {
+ unionIterator.add(templateTypeInfos[i].getClazzIterator());
+ }
+ }
+
+ return unionIterator;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+
+ builder.append(clazz.getSimpleName());
+
+ if (templateTypeInfos != null) {
+ builder.append("<");
+
+ for (int i = 0; i < templateTypeInfos.length - 1; i++) {
+ builder.append(templateTypeInfos[i]).append(", ");
+ }
+
+ builder.append(templateTypeInfos[templateTypeInfos.length - 1]);
+ builder.append(">");
+ }
+
+ return builder.toString();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof FullTypeInfo) {
+ FullTypeInfo other = (FullTypeInfo) obj;
+
+ return clazz == other.getClazz() && Arrays.equals(templateTypeInfos, other.getTemplateTypeInfos());
+ } else {
+ return false;
+ }
+ }
+ }
+
+ /**
* Private constructor to prevent instantiation.
*/
private ReflectionUtil() {
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
index 857628f..1093a5a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
@@ -55,7 +55,7 @@ public final class TaskDeploymentDescriptor implements Serializable {
private final ExecutionAttemptID executionId;
/** The allocation ID of the slot in which the task shall be run */
- private final AllocationID allocationID;
+ private final AllocationID allocationId;
/** The task's name. */
private final String taskName;
@@ -105,6 +105,7 @@ public final class TaskDeploymentDescriptor implements Serializable {
*/
public TaskDeploymentDescriptor(
JobID jobID,
+ AllocationID allocationId,
String jobName,
JobVertexID vertexID,
ExecutionAttemptID executionId,
@@ -130,6 +131,7 @@ public final class TaskDeploymentDescriptor implements Serializable {
checkArgument(attemptNumber >= 0);
this.jobID = checkNotNull(jobID);
+ this.allocationId = checkNotNull(allocationId);
this.jobName = checkNotNull(jobName);
this.vertexID = checkNotNull(vertexID);
this.executionId = checkNotNull(executionId);
@@ -148,11 +150,11 @@ public final class TaskDeploymentDescriptor implements Serializable {
this.requiredClasspaths = checkNotNull(requiredClasspaths);
this.targetSlotNumber = targetSlotNumber;
this.taskStateHandles = taskStateHandles;
- this.allocationID = new AllocationID();
}
public TaskDeploymentDescriptor(
JobID jobID,
+ AllocationID allocationId,
String jobName,
JobVertexID vertexID,
ExecutionAttemptID executionId,
@@ -173,6 +175,7 @@ public final class TaskDeploymentDescriptor implements Serializable {
this(
jobID,
+ allocationId,
jobName,
vertexID,
executionId,
@@ -311,8 +314,8 @@ public final class TaskDeploymentDescriptor implements Serializable {
return requiredClasspaths;
}
- public AllocationID getAllocationID() {
- return allocationID;
+ public AllocationID getAllocationId() {
+ return allocationId;
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
index 8979d7c..eea2e81 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
@@ -24,6 +24,7 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.JobException;
import org.apache.flink.runtime.blob.BlobKey;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.deployment.InputChannelDeploymentDescriptor;
import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
import org.apache.flink.runtime.deployment.PartialInputChannelDeploymentDescriptor;
@@ -605,6 +606,7 @@ public class ExecutionVertex implements AccessExecutionVertex, Archiveable<Archi
return new TaskDeploymentDescriptor(
getJobId(),
+ new AllocationID(), // TODO: Obtain the proper allocation id from the slot
getExecutionGraph().getJobName(),
getJobvertexId(),
executionId,
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
index 5d78ffc..484cddb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
@@ -49,10 +49,11 @@ public interface HighAvailabilityServices {
* Gets the leader retriever for the job JobMaster which is responsible for the given job
*
* @param jobID The identifier of the job.
+ * @param defaultAddress address under which the job manager is reachable
* @return
* @throws Exception
*/
- LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception;
+ LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultAddress) throws Exception;
/**
* Gets the leader election service for the cluster's resource manager.
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
index d7fd2bf..1c73c01 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
@@ -82,8 +82,8 @@ public class NonHaServices implements HighAvailabilityServices {
}
@Override
- public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception {
- return new StandaloneLeaderRetrievalService(jobMastersAddress.get(jobID), new UUID(0, 0));
+ public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultAddress) throws Exception {
+ return new StandaloneLeaderRetrievalService(defaultAddress, new UUID(0, 0));
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
index 3a7736b..bbe8ecb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
@@ -111,7 +111,7 @@ public class ZookeeperHaServices implements HighAvailabilityServices {
}
@Override
- public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception {
+ public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultAddress) throws Exception {
return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, getPathForJobManager(jobID));
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JMTMRegistrationSuccess.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JMTMRegistrationSuccess.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JMTMRegistrationSuccess.java
new file mode 100644
index 0000000..7272cd4
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JMTMRegistrationSuccess.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.registration.RegistrationResponse;
+import org.apache.flink.util.Preconditions;
+
+public class JMTMRegistrationSuccess extends RegistrationResponse.Success {
+ private static final long serialVersionUID = -3528383155961318929L;
+
+ private final ResourceID resourceID;
+ private final int blobPort;
+
+ public JMTMRegistrationSuccess(ResourceID resourceID, int blobPort) {
+ Preconditions.checkArgument(0 < blobPort && 65536 > blobPort, "The blob port has to be 0 < blobPort < 65536.");
+
+ this.resourceID = Preconditions.checkNotNull(resourceID);
+ this.blobPort = blobPort;
+ }
+
+ public ResourceID getResourceID() {
+ return resourceID;
+ }
+
+ public int getBlobPort() {
+ return blobPort;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index e11f3a1..a7be476 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -33,6 +33,7 @@ import org.apache.flink.runtime.checkpoint.CompletedCheckpoint;
import org.apache.flink.runtime.client.JobExecutionException;
import org.apache.flink.runtime.client.JobSubmissionException;
import org.apache.flink.runtime.client.SerializedJobExecutionResult;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.concurrent.ApplyFunction;
import org.apache.flink.runtime.concurrent.Future;
@@ -646,6 +647,24 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
executionGraph.getRequiredClasspaths());
}
+ @RpcMethod
+ public Iterable<AllocationID> offerSlots(final Iterable<AllocationID> slots, UUID leaderId) {
+ throw new UnsupportedOperationException("Has to be implemented.");
+ }
+
+ @RpcMethod
+ public void failSlot(final AllocationID allocationId, UUID leaderId, Exception cause) {
+ throw new UnsupportedOperationException("Has to be implemented.");
+ }
+
+ @RpcMethod
+ public RegistrationResponse registerTaskManager(
+ final String taskManagerAddress,
+ final ResourceID taskManagerProcessId,
+ final UUID leaderId) {
+ throw new UnsupportedOperationException("Has to be implemented.");
+ }
+
//----------------------------------------------------------------------------------------------
// Internal methods
//----------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
index b27b41c..0f155a4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
@@ -20,6 +20,7 @@ package org.apache.flink.runtime.jobmaster;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.checkpoint.CheckpointCoordinatorGateway;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
@@ -35,6 +36,7 @@ import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.query.KvStateID;
import org.apache.flink.runtime.query.KvStateLocation;
import org.apache.flink.runtime.query.KvStateServerAddress;
+import org.apache.flink.runtime.registration.RegistrationResponse;
import org.apache.flink.runtime.rpc.RpcTimeout;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.KvState;
@@ -170,4 +172,38 @@ public interface JobMasterGateway extends CheckpointCoordinatorGateway {
* Request the classloading props of this job.
*/
Future<ClassloadingProps> requestClassloadingProps();
+
+ /**
+ * Offer the given slots to the job manager. The response contains the set of accepted slots.
+ *
+ * @param slots to offer to the job manager
+ * @param leaderId identifying the job leader
+ * @param timeout for the rpc call
+ * @return Future set of accepted slots.
+ */
+ Future<Iterable<AllocationID>> offerSlots(final Iterable<AllocationID> slots, UUID leaderId, @RpcTimeout final Time timeout);
+
+ /**
+ * Fail the slot with the given allocation id and cause.
+ *
+ * @param allocationId identifying the slot to fail
+ * @param leaderId identifying the job leader
+ * @param cause of the failing
+ */
+ void failSlot(final AllocationID allocationId, UUID leaderId, Exception cause);
+
+ /**
+ * Register the task manager at the job manager.
+ *
+ * @param taskManagerAddress address of the task manager
+ * @param taskManagerProcessId identifying the task manager
+ * @param leaderId identifying the job leader
+ * @param timeout for the rpc call
+ * @return Future registration response indicating whether the registration was successful or not
+ */
+ Future<RegistrationResponse> registerTaskManager(
+ final String taskManagerAddress,
+ final ResourceID taskManagerProcessId,
+ final UUID leaderId,
+ @RpcTimeout final Time timeout);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/registration/RegisteredRpcConnection.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/registration/RegisteredRpcConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/registration/RegisteredRpcConnection.java
index 76093b0..78d4dbc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/registration/RegisteredRpcConnection.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/registration/RegisteredRpcConnection.java
@@ -35,7 +35,7 @@ import static org.apache.flink.util.Preconditions.checkState;
/**
* This utility class implements the basis of RPC connecting from one component to another component,
* for example the RPC connection from TaskExecutor to ResourceManager.
- * This {@code RegisteredRpcConnection} implements registration and get target gateway .
+ * This {@code RegisteredRpcConnection} implements registration and get target gateway.
*
* <p>The registration gives access to a future that is completed upon successful registration.
* The RPC connection can be closed, for example when the target where it tries to register
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index 3122804..6f6d525 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -188,7 +188,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
} else {
try {
LeaderRetrievalService jobMasterLeaderRetriever =
- highAvailabilityServices.getJobManagerLeaderRetriever(jobID);
+ highAvailabilityServices.getJobManagerLeaderRetriever(jobID, jobMasterAddress);
jobIdLeaderListener = new JobIdLeaderListener(jobID, jobMasterLeaderRetriever);
} catch (Exception e) {
log.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
index e312ea2..f055971 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
@@ -298,7 +298,13 @@ public abstract class SlotManager {
final TaskExecutorRegistration registration = freeSlot.getTaskExecutorRegistration();
final Future<TMSlotRequestReply> slotRequestReplyFuture =
registration.getTaskExecutorGateway()
- .requestSlot(freeSlot.getSlotId(), allocationID, rmServices.getLeaderID(), timeout);
+ .requestSlot(
+ freeSlot.getSlotId(),
+ slotRequest.getJobId(),
+ allocationID,
+ "foobar", // TODO: set proper JM address
+ rmServices.getLeaderID(),
+ timeout);
slotRequestReplyFuture.handleAsync(new BiFunction<TMSlotRequestReply, Throwable, Void>() {
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderListener.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderListener.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderListener.java
new file mode 100644
index 0000000..f02a8c2
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderListener.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.jobmaster.JMTMRegistrationSuccess;
+import org.apache.flink.runtime.jobmaster.JobMasterGateway;
+
+import java.util.UUID;
+
+/**
+ * Listener for the {@link JobLeaderService}. The listener is notified whenever a job manager
+ * gained leadership for a registered job and the service could establish a connection to it.
+ * Furthermore, the listener is notified when a job manager loses leadership for a job. In case
+ * of an error, the {@link #handleError(Throwable)}} is called.
+ */
+public interface JobLeaderListener {
+
+ /**
+ * Callback if a job manager has gained leadership for the job identified by the job id and a
+ * connection could be established to this job manager.
+ *
+ * @param jobId identifying the job for which the job manager has gained leadership
+ * @param jobManagerGateway to the job leader
+ * @param jobLeaderId new leader id of the job leader
+ * @param registrationMessage containing further registration information
+ */
+ void jobManagerGainedLeadership(JobID jobId, JobMasterGateway jobManagerGateway, UUID jobLeaderId, JMTMRegistrationSuccess registrationMessage);
+
+ /**
+ * Callback if the job leader for the job with the given job id lost its leadership.
+ *
+ * @param jobId identifying the job whose leader has lost leadership
+ * @param jobLeaderId old leader id
+ */
+ void jobManagerLostLeadership(JobID jobId, UUID jobLeaderId);
+
+ /**
+ * Callback for errors which might occur in the {@link JobLeaderService}.
+ *
+ * @param throwable cause
+ */
+ void handleError(Throwable throwable);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
new file mode 100644
index 0000000..9e71349
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
@@ -0,0 +1,390 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.jobmaster.JMTMRegistrationSuccess;
+import org.apache.flink.runtime.jobmaster.JobMasterGateway;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.runtime.registration.RegisteredRpcConnection;
+import org.apache.flink.runtime.registration.RegistrationResponse;
+import org.apache.flink.runtime.registration.RetryingRegistration;
+import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Executor;
+
+/**
+ * This service has the responsibility to monitor the job leaders (the job manager which is leader
+ * for a given job) for all registered jobs. Upon gaining leadership for a job and detection by the
+ * job leader service, the service tries to establish a connection to the job leader. After
+ * successfully establishing a connection, the job leader listener is notified about the new job
+ * leader and its connection. In case that a job leader loses leadership, the job leader listener
+ * is notified as well.
+ */
+public class JobLeaderService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(JobLeaderService.class);
+
+ /** Process id of the owning process */
+ private final ResourceID ownerProcessId;
+
+ /** The leader retrieval service and listener for each registered job */
+ private final Map<JobID, Tuple2<LeaderRetrievalService, JobLeaderService.JobManagerLeaderListener>> jobLeaderServices;
+
+ /** Internal state of the service */
+ private volatile JobLeaderService.State state;
+
+ /** Address of the owner of this service. This address is used for the job manager connection */
+ private String ownerAddress;
+
+ /** Rpc service to use for establishing connections */
+ private RpcService rpcService;
+
+ /** High availability services to create the leader retrieval services from */
+ private HighAvailabilityServices highAvailabilityServices;
+
+ /** Job leader listener listening for job leader changes */
+ private JobLeaderListener jobLeaderListener;
+
+ public JobLeaderService(ResourceID ownerProcessId) {
+ this.ownerProcessId = Preconditions.checkNotNull(ownerProcessId);
+
+ jobLeaderServices = new HashMap<>(4);
+
+ state = JobLeaderService.State.CREATED;
+
+ ownerAddress = null;
+ rpcService = null;
+ highAvailabilityServices = null;
+ jobLeaderListener = null;
+ }
+
+ // -------------------------------------------------------------------------------
+ // Methods
+ // -------------------------------------------------------------------------------
+
+ /**
+ * Start the job leader service with the given services.
+ *
+ * @param initialOwnerAddress to be used for establishing connections (source address)
+ * @param initialRpcService to be used to create rpc connections
+ * @param initialHighAvailabilityServices to create leader retrieval services for the different jobs
+ * @param initialJobLeaderListener listening for job leader changes
+ */
+ public void start(
+ final String initialOwnerAddress,
+ final RpcService initialRpcService,
+ final HighAvailabilityServices initialHighAvailabilityServices,
+ final JobLeaderListener initialJobLeaderListener) {
+
+ if (JobLeaderService.State.CREATED != state) {
+ throw new IllegalStateException("The service has already been started.");
+ } else {
+ LOG.info("Start job leader service.");
+
+ this.ownerAddress = Preconditions.checkNotNull(initialOwnerAddress);
+ this.rpcService = Preconditions.checkNotNull(initialRpcService);
+ this.highAvailabilityServices = Preconditions.checkNotNull(initialHighAvailabilityServices);
+ this.jobLeaderListener = Preconditions.checkNotNull(initialJobLeaderListener);
+ state = JobLeaderService.State.STARTED;
+ }
+ }
+
+ /**
+ * Stop the job leader services. This implies stopping all leader retrieval services for the
+ * different jobs and their leader retrieval listeners.
+ *
+ * @throws Exception if an error occurs while stopping the service
+ */
+ public void stop() throws Exception {
+ LOG.info("Stop job leader service.");
+
+ if (JobLeaderService.State.STARTED == state) {
+
+ for (Tuple2<LeaderRetrievalService, JobLeaderService.JobManagerLeaderListener> leaderRetrievalServiceEntry: jobLeaderServices.values()) {
+ LeaderRetrievalService leaderRetrievalService = leaderRetrievalServiceEntry.f0;
+ JobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = leaderRetrievalServiceEntry.f1;
+
+ jobManagerLeaderListener.stop();
+ leaderRetrievalService.stop();
+ }
+
+ jobLeaderServices.clear();
+ }
+
+ state = JobLeaderService.State.STOPPED;
+ }
+
+ /**
+ * Check whether the service monitors the given job.
+ *
+ * @param jobId identifying the job
+ * @return True if the given job is monitored; otherwise false
+ */
+ public boolean containsJob(JobID jobId) {
+ Preconditions.checkState(JobLeaderService.State.STARTED == state, "The service is currently not running.");
+
+ return jobLeaderServices.containsKey(jobId);
+ }
+
+ /**
+ * Remove the given job from being monitored by the job leader service.
+ *
+ * @param jobId identifying the job to remove from monitoring
+ * @throws Exception if an error occurred while stopping the leader retrieval service and listener
+ */
+ public void removeJob(JobID jobId) throws Exception {
+ Preconditions.checkState(JobLeaderService.State.STARTED == state, "The service is currently not running.");
+
+ Tuple2<LeaderRetrievalService, JobLeaderService.JobManagerLeaderListener> entry = jobLeaderServices.remove(jobId);
+
+ if (entry != null) {
+ LOG.info("Remove job {} from job leader monitoring.", jobId);
+
+ LeaderRetrievalService leaderRetrievalService = entry.f0;
+ JobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = entry.f1;
+
+ leaderRetrievalService.stop();
+ jobManagerLeaderListener.stop();
+ }
+ }
+
+ /**
+ * Add the given job to be monitored. This means that the service tries to detect leaders for
+ * this job and then tries to establish a connection to it.
+ *
+ * @param jobId identifying the job to monitor
+ * @param defaultTargetAddress of the job leader
+ * @throws Exception if an error occurs while starting the leader retrieval service
+ */
+ public void addJob(final JobID jobId, final String defaultTargetAddress) throws Exception {
+ Preconditions.checkState(JobLeaderService.State.STARTED == state, "The service is currently not running.");
+
+ LOG.info("Add job {} for job leader monitoring.", jobId);
+
+ final LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(
+ jobId,
+ defaultTargetAddress);
+
+ JobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = new JobManagerLeaderListener(jobId);
+
+ leaderRetrievalService.start(jobManagerLeaderListener);
+
+ jobLeaderServices.put(jobId, Tuple2.of(leaderRetrievalService, jobManagerLeaderListener));
+ }
+
+ /**
+ * Leader listener which tries to establish a connection to a newly detected job leader.
+ */
+ private final class JobManagerLeaderListener implements LeaderRetrievalListener {
+
+ /** Job id identifying the job to look for a leader */
+ private final JobID jobId;
+
+ /** Rpc connection to the job leader */
+ private RegisteredRpcConnection<JobMasterGateway, JMTMRegistrationSuccess> rpcConnection;
+
+ /** State of the listener */
+ private volatile boolean stopped;
+
+ /** Leader id of the current job leader */
+ private volatile UUID currentLeaderId;
+
+ private JobManagerLeaderListener(JobID jobId) {
+ this.jobId = Preconditions.checkNotNull(jobId);
+
+ stopped = false;
+ rpcConnection = null;
+ currentLeaderId = null;
+ }
+
+ public void stop() {
+ stopped = true;
+
+ if (rpcConnection != null) {
+ rpcConnection.close();
+ }
+ }
+
+ @Override
+ public void notifyLeaderAddress(final String leaderAddress, final UUID leaderId) {
+ if (stopped) {
+ LOG.debug("{}'s leader retrieval listener reported a new leader for job {}. " +
+ "However, the service is no longer running.", JobLeaderService.class.getSimpleName(), jobId);
+ } else {
+ LOG.debug("New leader information for job {}. Address: {}, leader id: {}.",
+ jobId, leaderAddress, leaderId);
+
+ if (leaderAddress == null || leaderAddress.isEmpty()) {
+ // the leader lost leadership but there is no other leader yet.
+ if (rpcConnection != null) {
+ rpcConnection.close();
+ }
+
+ jobLeaderListener.jobManagerLostLeadership(jobId, currentLeaderId);
+
+ currentLeaderId = leaderId;
+ } else {
+ currentLeaderId = leaderId;
+
+ if (rpcConnection != null) {
+ // check if we are already trying to connect to this leader
+ if (!leaderId.equals(rpcConnection.getTargetLeaderId())) {
+ rpcConnection.close();
+
+ rpcConnection = new JobManagerRegisteredRpcConnection(
+ LOG,
+ leaderAddress,
+ leaderId,
+ rpcService.getExecutor());
+ }
+ } else {
+ rpcConnection = new JobManagerRegisteredRpcConnection(
+ LOG,
+ leaderAddress,
+ leaderId,
+ rpcService.getExecutor());
+ }
+
+ // double check for a concurrent stop operation
+ if (stopped) {
+ rpcConnection.close();
+ } else {
+ LOG.info("Try to register at job manager {} with leader id {}.", leaderAddress, leaderId);
+ rpcConnection.start();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void handleError(Exception exception) {
+ if (stopped) {
+ LOG.debug("{}'s leader retrieval listener reported an exception for job {}. " +
+ "However, the service is no longer running.", JobLeaderService.class.getSimpleName(),
+ jobId, exception);
+ } else {
+ jobLeaderListener.handleError(exception);
+ }
+ }
+
+ /**
+ * Rpc connection for the job manager <--> task manager connection.
+ */
+ private final class JobManagerRegisteredRpcConnection extends RegisteredRpcConnection<JobMasterGateway, JMTMRegistrationSuccess> {
+
+ JobManagerRegisteredRpcConnection(
+ Logger log,
+ String targetAddress,
+ UUID targetLeaderId,
+ Executor executor) {
+ super(log, targetAddress, targetLeaderId, executor);
+ }
+
+ @Override
+ protected RetryingRegistration<JobMasterGateway, JMTMRegistrationSuccess> generateRegistration() {
+ return new JobLeaderService.JobManagerRetryingRegistration(
+ LOG,
+ rpcService,
+ "JobManager",
+ JobMasterGateway.class,
+ getTargetAddress(),
+ getTargetLeaderId(),
+ ownerAddress,
+ ownerProcessId);
+ }
+
+ @Override
+ protected void onRegistrationSuccess(JMTMRegistrationSuccess success) {
+ // filter out old registration attempts
+ if (getTargetLeaderId().equals(currentLeaderId)) {
+ log.info("Successful registration at job manager {} for job {}.", getTargetAddress(), jobId);
+
+ jobLeaderListener.jobManagerGainedLeadership(jobId, getTargetGateway(), getTargetLeaderId(), success);
+ } else {
+ log.debug("Encountered obsolete JobManager registration success from {} with leader session ID {}.", getTargetAddress(), getTargetLeaderId());
+ }
+ }
+
+ @Override
+ protected void onRegistrationFailure(Throwable failure) {
+ // filter out old registration attempts
+ if (getTargetLeaderId().equals(currentLeaderId)) {
+ log.info("Failed to register at job manager {} for job {}.", getTargetAddress(), jobId);
+ jobLeaderListener.handleError(failure);
+ } else {
+ log.debug("Obsolete JobManager registration failure from {} with leader session ID {}.", getTargetAddress(), getTargetLeaderId(), failure);
+ }
+ }
+ }
+ }
+
+ /**
+ * Retrying registration for the job manager <--> task manager connection.
+ */
+ private static final class JobManagerRetryingRegistration extends RetryingRegistration<JobMasterGateway, JMTMRegistrationSuccess> {
+
+ private final String taskManagerAddress;
+ private final ResourceID taskManagerProcessId;
+
+ JobManagerRetryingRegistration(
+ Logger log,
+ RpcService rpcService,
+ String targetName,
+ Class<JobMasterGateway> targetType,
+ String targetAddress,
+ UUID leaderId,
+ String taskManagerAddress,
+ ResourceID taskManagerProcessId) {
+
+ super(log, rpcService, targetName, targetType, targetAddress, leaderId);
+
+ this.taskManagerAddress = Preconditions.checkNotNull(taskManagerAddress);
+ this.taskManagerProcessId = Preconditions.checkNotNull(taskManagerProcessId);
+ }
+
+ @Override
+ protected Future<RegistrationResponse> invokeRegistration(JobMasterGateway gateway, UUID leaderId, long timeoutMillis) throws Exception {
+ return gateway.registerTaskManager(
+ taskManagerAddress,
+ taskManagerProcessId,
+ leaderId,
+ Time.milliseconds(timeoutMillis));
+ }
+ }
+
+ /**
+ * Internal state of the service
+ */
+ private enum State {
+ CREATED, STARTED, STOPPED
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerConnection.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerConnection.java
index 6fcd082..8d2057a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerConnection.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerConnection.java
@@ -34,7 +34,7 @@ import java.util.UUID;
public class JobManagerConnection {
// Job master leader session id
- private final UUID jobMasterLeaderId;
+ private final UUID leaderId;
// Gateway to the job master
private final JobMasterGateway jobMasterGateway;
@@ -55,15 +55,14 @@ public class JobManagerConnection {
private final PartitionStateChecker partitionStateChecker;
public JobManagerConnection(
- UUID jobMasterLeaderId,
- JobMasterGateway jobMasterGateway,
- TaskManagerActions taskManagerActions,
- CheckpointResponder checkpointResponder,
- LibraryCacheManager libraryCacheManager,
- ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
- PartitionStateChecker partitionStateChecker)
- {
- this.jobMasterLeaderId = Preconditions.checkNotNull(jobMasterLeaderId);
+ JobMasterGateway jobMasterGateway,
+ UUID leaderId,
+ TaskManagerActions taskManagerActions,
+ CheckpointResponder checkpointResponder,
+ LibraryCacheManager libraryCacheManager,
+ ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
+ PartitionStateChecker partitionStateChecker) {
+ this.leaderId = Preconditions.checkNotNull(leaderId);
this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
this.taskManagerActions = Preconditions.checkNotNull(taskManagerActions);
this.checkpointResponder = Preconditions.checkNotNull(checkpointResponder);
@@ -72,8 +71,8 @@ public class JobManagerConnection {
this.partitionStateChecker = Preconditions.checkNotNull(partitionStateChecker);
}
- public UUID getJobMasterLeaderId() {
- return jobMasterLeaderId;
+ public UUID getLeaderId() {
+ return leaderId;
}
public JobMasterGateway getJobManagerGateway() {
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerTable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerTable.java
new file mode 100644
index 0000000..00c467e
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerTable.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor;
+
+import org.apache.flink.api.common.JobID;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Container for multiple {@link JobManagerConnection} registered under their respective job id.
+ */
+public class JobManagerTable {
+ private final Map<JobID, JobManagerConnection> jobManagerConnections;
+
+ public JobManagerTable() {
+ jobManagerConnections = new HashMap<>(4);
+ }
+
+ public boolean contains(JobID jobId) {
+ return jobManagerConnections.containsKey(jobId);
+ }
+
+ public boolean put(JobID jobId, JobManagerConnection jobManagerConnection) {
+ JobManagerConnection previousJMC = jobManagerConnections.put(jobId, jobManagerConnection);
+
+ if (previousJMC != null) {
+ jobManagerConnections.put(jobId, previousJMC);
+
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ public JobManagerConnection remove(JobID jobId) {
+ return jobManagerConnections.remove(jobId);
+ }
+
+ public JobManagerConnection get(JobID jobId) {
+ return jobManagerConnections.get(jobId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index e642315..3e3a544 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -19,12 +19,14 @@
package org.apache.flink.runtime.taskexecutor;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.accumulators.AccumulatorSnapshot;
import org.apache.flink.runtime.blob.BlobCache;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.clusterframework.types.SlotID;
+import org.apache.flink.runtime.concurrent.AcceptFunction;
import org.apache.flink.runtime.concurrent.ApplyFunction;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
@@ -34,7 +36,6 @@ import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.executiongraph.PartitionInfo;
import org.apache.flink.runtime.filecache.FileCache;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.instance.InstanceID;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.network.NetworkEnvironment;
import org.apache.flink.runtime.io.network.netty.PartitionStateChecker;
@@ -42,6 +43,7 @@ import org.apache.flink.runtime.io.network.partition.ResultPartitionConsumableNo
import org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
+import org.apache.flink.runtime.jobmaster.JMTMRegistrationSuccess;
import org.apache.flink.runtime.jobmaster.JobMasterGateway;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
import org.apache.flink.runtime.memory.MemoryManager;
@@ -51,7 +53,6 @@ import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRegistered;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRejected;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcEndpoint;
@@ -59,6 +60,7 @@ import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.taskexecutor.exceptions.CheckpointException;
import org.apache.flink.runtime.taskexecutor.exceptions.PartitionException;
+import org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException;
import org.apache.flink.runtime.taskexecutor.exceptions.TaskException;
import org.apache.flink.runtime.taskexecutor.exceptions.TaskSubmissionException;
import org.apache.flink.runtime.taskexecutor.rpc.RpcCheckpointResponder;
@@ -81,9 +83,10 @@ import java.net.InetSocketAddress;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.Map;
-import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.TimeoutException;
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -97,6 +100,9 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
/** The connection information of this task manager */
private final TaskManagerLocation taskManagerLocation;
+ /** Max blob port which is accepted */
+ public static final int MAX_BLOB_PORT = 65536;
+
/** The access to the leader election and retrieval services */
private final HighAvailabilityServices haServices;
@@ -121,10 +127,6 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
private final TaskManagerMetricGroup taskManagerMetricGroup;
private final BroadcastVariableManager broadcastVariableManager;
-
- /** Slots which have become available but haven't been confirmed by the RM */
- private final Set<SlotID> unconfirmedFreeSlots;
-
private final FileCache fileCache;
@@ -140,6 +142,10 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
private final TaskSlotTable taskSlotTable;
+ private final JobManagerTable jobManagerTable;
+
+ private final JobLeaderService jobLeaderService;
+
// ------------------------------------------------------------------------
public TaskExecutor(
@@ -155,6 +161,8 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
BroadcastVariableManager broadcastVariableManager,
FileCache fileCache,
TaskSlotTable taskSlotTable,
+ JobManagerTable jobManagerTable,
+ JobLeaderService jobLeaderService,
FatalErrorHandler fatalErrorHandler) {
super(rpcService);
@@ -173,10 +181,10 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
this.taskManagerMetricGroup = checkNotNull(taskManagerMetricGroup);
this.broadcastVariableManager = checkNotNull(broadcastVariableManager);
this.fileCache = checkNotNull(fileCache);
+ this.jobManagerTable = checkNotNull(jobManagerTable);
+ this.jobLeaderService = checkNotNull(jobLeaderService);
this.jobManagerConnections = new HashMap<>(4);
-
- this.unconfirmedFreeSlots = new HashSet<>();
}
// ------------------------------------------------------------------------
@@ -195,7 +203,10 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
// tell the task slot table who's responsible for the task slot actions
- taskSlotTable.start(new SlotActionsImpl(), taskManagerConfiguration.getTimeout());
+ taskSlotTable.start(new SlotActionsImpl());
+
+ // start the job leader service
+ jobLeaderService.start(getAddress(), getRpcService(), haServices, new JobLeaderListenerImpl());
}
/**
@@ -207,7 +218,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
taskSlotTable.stop();
- if (resourceManagerConnection.isConnected()) {
+ if (isConnectedToResourceManager()) {
try {
resourceManagerConnection.close();
} catch (Exception e) {
@@ -248,30 +259,39 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
log.info("Stopped TaskManager {}.", getAddress());
}
- // ========================================================================
+ // ======================================================================
// RPC methods
- // ========================================================================
+ // ======================================================================
// ----------------------------------------------------------------------
// Task lifecycle RPCs
// ----------------------------------------------------------------------
@RpcMethod
- public Acknowledge submitTask(TaskDeploymentDescriptor tdd, ResourceID jobManagerID) throws TaskSubmissionException {
+ public Acknowledge submitTask(TaskDeploymentDescriptor tdd, UUID jobManagerLeaderId) throws TaskSubmissionException {
- JobManagerConnection jobManagerConnection = getJobManagerConnection(jobManagerID);
+ JobManagerConnection jobManagerConnection = jobManagerTable.get(tdd.getJobID());
if (jobManagerConnection == null) {
- final String message = "Could not submit task because JobManager " + jobManagerID +
- " was not associated.";
+ final String message = "Could not submit task because there is no JobManager " +
+ "associated for the job " + tdd.getJobID() + '.';
+
+ log.debug(message);
+ throw new TaskSubmissionException(message);
+ }
+
+ if (!jobManagerConnection.getLeaderId().equals(jobManagerLeaderId)) {
+ final String message = "Rejecting the task submission because the job manager leader id " +
+ jobManagerLeaderId + " does not match the expected job manager leader id " +
+ jobManagerConnection.getLeaderId() + '.';
log.debug(message);
throw new TaskSubmissionException(message);
}
- if (!taskSlotTable.existActiveSlot(tdd.getJobID(), tdd.getAllocationID())) {
+ if (!taskSlotTable.existsActiveSlot(tdd.getJobID(), tdd.getAllocationId())) {
final String message = "No task slot allocated for job ID " + tdd.getJobID() +
- " and allocation ID " + tdd.getAllocationID() + '.';
+ " and allocation ID " + tdd.getAllocationId() + '.';
log.debug(message);
throw new TaskSubmissionException(message);
}
@@ -279,7 +299,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
TaskMetricGroup taskMetricGroup = taskManagerMetricGroup.addTaskForJob(tdd);
InputSplitProvider inputSplitProvider = new RpcInputSplitProvider(
- jobManagerConnection.getJobMasterLeaderId(),
+ jobManagerConnection.getLeaderId(),
jobManagerConnection.getJobManagerGateway(),
tdd.getJobID(),
tdd.getVertexID(),
@@ -375,7 +395,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
// ----------------------------------------------------------------------
@RpcMethod
- public Acknowledge updatePartitions(final ExecutionAttemptID executionAttemptID, Collection<PartitionInfo> partitionInfos) throws PartitionException {
+ public Acknowledge updatePartitions(final ExecutionAttemptID executionAttemptID, Iterable<PartitionInfo> partitionInfos) throws PartitionException {
final Task task = taskSlotTable.getTask(executionAttemptID);
if (task != null) {
@@ -471,38 +491,319 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
+ // ----------------------------------------------------------------------
+ // Slot allocation RPCs
+ // ----------------------------------------------------------------------
+
/**
+ * /**
* Requests a slot from the TaskManager
*
- * @param slotID Slot id for the request
- * @param allocationID id for the request
- * @param resourceManagerLeaderID current leader id of the ResourceManager
+ * @param slotId identifying the requested slot
+ * @param jobId identifying the job for which the request is issued
+ * @param allocationId id for the request
+ * @param targetAddress of the job manager requesting the slot
+ * @param rmLeaderId current leader id of the ResourceManager
+ * @throws SlotAllocationException if the slot allocation fails
* @return answer to the slot request
*/
@RpcMethod
- public TMSlotRequestReply requestSlot(SlotID slotID, AllocationID allocationID, UUID resourceManagerLeaderID) {
- if (!resourceManagerConnection.getTargetLeaderId().equals(resourceManagerLeaderID)) {
- return new TMSlotRequestRejected(
- resourceManagerConnection.getRegistrationId(), getResourceID(), allocationID);
+ public TMSlotRequestReply requestSlot(
+ final SlotID slotId,
+ final JobID jobId,
+ final AllocationID allocationId,
+ final String targetAddress,
+ final UUID rmLeaderId) throws SlotAllocationException {
+ log.info("Receive slot request {} for job {} from resource manager with leader id {}.",
+ allocationId, jobId, rmLeaderId);
+
+ if (resourceManagerConnection == null) {
+ final String message = "TaskManager is not connected to a resource manager.";
+ log.debug(message);
+ throw new SlotAllocationException(message);
}
- if (unconfirmedFreeSlots.contains(slotID)) {
- // check if request has not been blacklisted because the notification of a free slot
- // has not been confirmed by the ResourceManager
- return new TMSlotRequestRejected(
- resourceManagerConnection.getRegistrationId(), getResourceID(), allocationID);
+
+ if (!resourceManagerConnection.getTargetLeaderId().equals(rmLeaderId)) {
+ final String message = "The leader id " + rmLeaderId +
+ " does not match with the leader id of the connected resource manager " +
+ resourceManagerConnection.getTargetLeaderId() + '.';
+
+ log.debug(message);
+ throw new SlotAllocationException(message);
+ }
+
+ if (taskSlotTable.isSlotFree(slotId.getSlotNumber())) {
+ if (taskSlotTable.allocateSlot(slotId.getSlotNumber(), jobId, allocationId, taskManagerConfiguration.getTimeout())) {
+ log.info("Allocated slot for {}.", allocationId);
+ } else {
+ log.info("Could not allocate slot for {}.", allocationId);
+ throw new SlotAllocationException("Could not allocate slot.");
+ }
+ } else if (!taskSlotTable.isAllocated(slotId.getSlotNumber(), jobId, allocationId)) {
+ final String message = "The slot " + slotId + " has already been allocated for a different job.";
+
+ log.info(message);
+ throw new SlotAllocationException(message);
+ }
+
+ if (jobManagerTable.contains(jobId)) {
+ offerSlotsToJobManager(jobId);
+ } else {
+ try {
+ jobLeaderService.addJob(jobId, targetAddress);
+ } catch (Exception e) {
+ // free the allocated slot
+ try {
+ taskSlotTable.freeSlot(allocationId);
+ } catch (SlotNotFoundException slotNotFoundException) {
+ // slot no longer existent, this should actually never happen, because we've
+ // just allocated the slot. So let's fail hard in this case!
+ onFatalError(slotNotFoundException);
+ }
+
+ // sanity check
+ if (!taskSlotTable.isSlotFree(slotId.getSlotNumber())) {
+ onFatalError(new Exception("Could not free slot " + slotId));
+ }
+
+ throw new SlotAllocationException("Could not add job to job leader service.", e);
+ }
}
- return new TMSlotRequestRegistered(new InstanceID(), ResourceID.generate(), allocationID);
+ return new TMSlotRequestRegistered(resourceManagerConnection.getRegistrationId(), getResourceID(), allocationId);
}
- // ------------------------------------------------------------------------
+ // ======================================================================
// Internal methods
+ // ======================================================================
+
+ // ------------------------------------------------------------------------
+ // Internal resource manager connection methods
+ // ------------------------------------------------------------------------
+
+ private void notifyOfNewResourceManagerLeader(String newLeaderAddress, UUID newLeaderId) {
+ if (resourceManagerConnection != null) {
+ if (newLeaderAddress != null) {
+ // the resource manager switched to a new leader
+ log.info("ResourceManager leader changed from {} to {}. Registering at new leader.",
+ resourceManagerConnection.getTargetAddress(), newLeaderAddress);
+ }
+ else {
+ // address null means that the current leader is lost without a new leader being there, yet
+ log.info("Current ResourceManager {} lost leader status. Waiting for new ResourceManager leader.",
+ resourceManagerConnection.getTargetAddress());
+ }
+
+ // drop the current connection or connection attempt
+ if (resourceManagerConnection != null) {
+ resourceManagerConnection.close();
+ resourceManagerConnection = null;
+ }
+ }
+
+ // establish a connection to the new leader
+ if (newLeaderAddress != null) {
+ log.info("Attempting to register at ResourceManager {}", newLeaderAddress);
+ resourceManagerConnection =
+ new TaskExecutorToResourceManagerConnection(
+ log,
+ this,
+ newLeaderAddress,
+ newLeaderId,
+ getMainThreadExecutor());
+ resourceManagerConnection.start();
+ }
+ }
+
// ------------------------------------------------------------------------
+ // Internal job manager connection methods
+ // ------------------------------------------------------------------------
+
+ private void offerSlotsToJobManager(final JobID jobId) {
+ final JobManagerConnection jobManagerConnection = jobManagerTable.get(jobId);
+
+ if (jobManagerConnection == null) {
+ log.debug("There is no job manager connection to the leader of job {}.", jobId);
+ } else {
+ if (taskSlotTable.hasAllocatedSlots(jobId)) {
+ log.info("Offer reserved slots to the leader of job {}.", jobId);
+
+ final JobMasterGateway jobMasterGateway = jobManagerConnection.getJobManagerGateway();
+
+ final Iterator<AllocationID> reservedSlotsIterator = taskSlotTable.getAllocatedSlots(jobId);
+ final UUID leaderId = jobManagerConnection.getLeaderId();
+
+ final Collection<AllocationID> reservedSlots = new HashSet<>(2);
+
+ while (reservedSlotsIterator.hasNext()) {
+ reservedSlots.add(reservedSlotsIterator.next());
+ }
+
+ Future<Iterable<AllocationID>> acceptedSlotsFuture = jobMasterGateway.offerSlots(
+ reservedSlots,
+ leaderId,
+ taskManagerConfiguration.getTimeout());
+
+ acceptedSlotsFuture.thenAcceptAsync(new AcceptFunction<Iterable<AllocationID>>() {
+ @Override
+ public void accept(Iterable<AllocationID> acceptedSlots) {
+ // check if the response is still valid
+ if (isJobManagerConnectionValid(jobId, leaderId)) {
+ // mark accepted slots active
+ for (AllocationID acceptedSlot: acceptedSlots) {
+ try {
+ if (!taskSlotTable.markSlotActive(acceptedSlot)) {
+ // the slot is either free or releasing at the moment
+ final String message = "Could not mark slot " + jobId + " active.";
+ log.debug(message);
+ jobMasterGateway.failSlot(acceptedSlot, leaderId, new Exception(message));
+ }
+
+ // remove the assigned slots so that we can free the left overs
+ reservedSlots.remove(acceptedSlot);
+ } catch (SlotNotFoundException e) {
+ log.debug("Could not mark slot {} active.", acceptedSlot, e);
+ jobMasterGateway.failSlot(acceptedSlot, leaderId, e);
+ }
+ }
+
+ final Exception e = new Exception("The slot was rejected by the JobManager.");
+
+ for (AllocationID rejectedSlot: reservedSlots) {
+ freeSlot(rejectedSlot, e);
+ }
+ } else {
+ // discard the response since there is a new leader for the job
+ log.debug("Discard offer slot response since there is a new leader " +
+ "for the job {}.", jobId);
+ }
+ }
+ }, getMainThreadExecutor());
+
+ acceptedSlotsFuture.exceptionallyAsync(new ApplyFunction<Throwable, Void>() {
+ @Override
+ public Void apply(Throwable throwable) {
+ if (throwable instanceof TimeoutException) {
+ // We ran into a timeout. Try again.
+ offerSlotsToJobManager(jobId);
+ } else {
+ // We encountered an exception. Free the slots and return them to the RM.
+ for (AllocationID reservedSlot: reservedSlots) {
+ freeSlot(reservedSlot, throwable);
+ }
+ }
+
+ return null;
+ }
+ }, getMainThreadExecutor());
+ } else {
+ log.debug("There are no unassigned slots for the job {}.", jobId);
+ }
+ }
+ }
+
+ private void establishJobManagerConnection(JobID jobId, JobMasterGateway jobMasterGateway, UUID jobManagerLeaderId, JMTMRegistrationSuccess registrationSuccess) {
+ log.info("Establish JobManager connection for job {}.", jobId);
+
+ if (jobManagerTable.contains(jobId)) {
+ JobManagerConnection oldJobManagerConnection = jobManagerTable.get(jobId);
+
+ if (!oldJobManagerConnection.getLeaderId().equals(jobManagerLeaderId)) {
+ closeJobManagerConnection(jobId);
+ jobManagerTable.put(jobId, associateWithJobManager(jobMasterGateway, jobManagerLeaderId, registrationSuccess.getBlobPort()));
+ }
+ } else {
+ jobManagerTable.put(jobId, associateWithJobManager(jobMasterGateway, jobManagerLeaderId, registrationSuccess.getBlobPort()));
+ }
- private JobManagerConnection getJobManagerConnection(ResourceID jobManagerID) {
- return jobManagerConnections.get(jobManagerID);
+ offerSlotsToJobManager(jobId);
}
+ private void closeJobManagerConnection(JobID jobId) {
+ log.info("Close JobManager connection for job {}.", jobId);
+
+ // 1. fail tasks running under this JobID
+ Iterator<Task> tasks = taskSlotTable.getTasks(jobId);
+
+ while (tasks.hasNext()) {
+ tasks.next().failExternally(new Exception("JobManager responsible for " + jobId +
+ " lost the leadership."));
+ }
+
+ // 2. Move the active slots to state allocated (possible to time out again)
+ Iterator<AllocationID> activeSlots = taskSlotTable.getActiveSlots(jobId);
+
+ while (activeSlots.hasNext()) {
+ AllocationID activeSlot = activeSlots.next();
+
+ try {
+ if (!taskSlotTable.markSlotInactive(activeSlot, taskManagerConfiguration.getTimeout())) {
+ freeSlot(activeSlot, new Exception("Slot could not be marked inactive."));
+ }
+ } catch (SlotNotFoundException e) {
+ log.debug("Could not mark the slot {} inactive.", jobId, e);
+ }
+ }
+
+ // 3. Disassociate from the JobManager
+ JobManagerConnection jobManagerConnection = jobManagerTable.remove(jobId);
+
+ if (jobManagerConnection != null) {
+ try {
+ disassociateFromJobManager(jobManagerConnection);
+ } catch (IOException e) {
+ log.warn("Could not properly disassociate from JobManager {}.",
+ jobManagerConnection.getJobManagerGateway().getAddress(), e);
+ }
+ }
+ }
+
+ private JobManagerConnection associateWithJobManager(JobMasterGateway jobMasterGateway, UUID jobManagerLeaderId, int blobPort) {
+ Preconditions.checkNotNull(jobManagerLeaderId);
+ Preconditions.checkNotNull(jobMasterGateway);
+ Preconditions.checkArgument(blobPort > 0 || blobPort < MAX_BLOB_PORT, "Blob port is out of range.");
+
+ TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobManagerLeaderId, jobMasterGateway);
+
+ CheckpointResponder checkpointResponder = new RpcCheckpointResponder(jobMasterGateway);
+
+ InetSocketAddress address = new InetSocketAddress(jobMasterGateway.getAddress(), blobPort);
+
+ BlobCache blobCache = new BlobCache(address, taskManagerConfiguration.getConfiguration());
+
+ LibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(
+ blobCache,
+ taskManagerConfiguration.getCleanupInterval());
+
+ ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = new RpcResultPartitionConsumableNotifier(
+ jobManagerLeaderId,
+ jobMasterGateway,
+ getRpcService().getExecutor(),
+ taskManagerConfiguration.getTimeout());
+
+ PartitionStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobManagerLeaderId, jobMasterGateway);
+
+ return new JobManagerConnection(
+ jobMasterGateway,
+ jobManagerLeaderId,
+ taskManagerActions,
+ checkpointResponder,
+ libraryCacheManager,
+ resultPartitionConsumableNotifier,
+ partitionStateChecker);
+ }
+
+ private void disassociateFromJobManager(JobManagerConnection jobManagerConnection) throws IOException {
+ Preconditions.checkNotNull(jobManagerConnection);
+ JobMasterGateway jobManagerGateway = jobManagerConnection.getJobManagerGateway();
+ jobManagerGateway.disconnectTaskManager(getResourceID());
+ jobManagerConnection.getLibraryCacheManager().shutdown();
+ }
+
+ // ------------------------------------------------------------------------
+ // Internal task methods
+ // ------------------------------------------------------------------------
+
private void failTask(final ExecutionAttemptID executionAttemptID, final Throwable cause) {
final Task task = taskSlotTable.getTask(executionAttemptID);
@@ -571,94 +872,11 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
- private void notifyOfNewResourceManagerLeader(String newLeaderAddress, UUID newLeaderId) {
- if (resourceManagerConnection != null) {
- if (newLeaderAddress != null) {
- // the resource manager switched to a new leader
- log.info("ResourceManager leader changed from {} to {}. Registering at new leader.",
- resourceManagerConnection.getTargetAddress(), newLeaderAddress);
- }
- else {
- // address null means that the current leader is lost without a new leader being there, yet
- log.info("Current ResourceManager {} lost leader status. Waiting for new ResourceManager leader.",
- resourceManagerConnection.getTargetAddress());
- }
-
- // drop the current connection or connection attempt
- if (resourceManagerConnection != null) {
- resourceManagerConnection.close();
- resourceManagerConnection = null;
- }
- }
-
- unconfirmedFreeSlots.clear();
-
- // establish a connection to the new leader
- if (newLeaderAddress != null) {
- log.info("Attempting to register at ResourceManager {}", newLeaderAddress);
- resourceManagerConnection =
- new TaskExecutorToResourceManagerConnection(
- log,
- this,
- newLeaderAddress,
- newLeaderId,
- getMainThreadExecutor());
- resourceManagerConnection.start();
- }
- }
-
- private JobManagerConnection associateWithJobManager(UUID jobMasterLeaderId,
- JobMasterGateway jobMasterGateway, int blobPort)
- {
- Preconditions.checkNotNull(jobMasterLeaderId);
- Preconditions.checkNotNull(jobMasterGateway);
- Preconditions.checkArgument(blobPort > 0 || blobPort <= 65535, "Blob port is out of range.");
-
- TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobMasterLeaderId, jobMasterGateway);
-
- CheckpointResponder checkpointResponder = new RpcCheckpointResponder(jobMasterGateway);
-
- InetSocketAddress address = new InetSocketAddress(jobMasterGateway.getAddress(), blobPort);
-
- BlobCache blobCache = new BlobCache(address, taskManagerConfiguration.getConfiguration());
-
- LibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(
- blobCache,
- taskManagerConfiguration.getCleanupInterval());
-
- ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = new RpcResultPartitionConsumableNotifier(
- jobMasterLeaderId,
- jobMasterGateway,
- getRpcService().getExecutor(),
- taskManagerConfiguration.getTimeout());
-
- PartitionStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobMasterLeaderId, jobMasterGateway);
-
- return new JobManagerConnection(
- jobMasterLeaderId,
- jobMasterGateway,
- taskManagerActions,
- checkpointResponder,
- libraryCacheManager,
- resultPartitionConsumableNotifier,
- partitionStateChecker);
- }
-
- private void disassociateFromJobManager(JobManagerConnection jobManagerConnection) throws IOException {
- if (jobManagerConnection != null) {
- JobMasterGateway jobManagerGateway = jobManagerConnection.getJobManagerGateway();
-
- jobManagerGateway.disconnectTaskManager(getResourceID());
-
- jobManagerConnection.getLibraryCacheManager().shutdown();
- }
- }
-
- private void freeSlot(AllocationID allocationId) {
+ private void freeSlot(AllocationID allocationId, Throwable cause) {
Preconditions.checkNotNull(allocationId);
try {
- int freedSlotIndex = taskSlotTable.freeSlot(allocationId);
+ int freedSlotIndex = taskSlotTable.freeSlot(allocationId, cause);
if (freedSlotIndex != -1 && isConnectedToResourceManager()) {
// the slot was freed. Tell the RM about it
@@ -674,21 +892,35 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
+ private void freeSlot(AllocationID allocationId) {
+ freeSlot(allocationId, new Exception("The slot " + allocationId + " is beeing freed."));
+ }
+
private void timeoutSlot(AllocationID allocationId, UUID ticket) {
Preconditions.checkNotNull(allocationId);
Preconditions.checkNotNull(ticket);
if (taskSlotTable.isValidTimeout(allocationId, ticket)) {
- freeSlot(allocationId);
+ freeSlot(allocationId, new Exception("The slot " + allocationId + " has timed out."));
} else {
log.debug("Received an invalid timeout for allocation id {} with ticket {}.", allocationId, ticket);
}
}
+ // ------------------------------------------------------------------------
+ // Internal utility methods
+ // ------------------------------------------------------------------------
+
private boolean isConnectedToResourceManager() {
return (resourceManagerConnection != null && resourceManagerConnection.isConnected());
}
+ private boolean isJobManagerConnectionValid(JobID jobId, UUID leaderId) {
+ JobManagerConnection jmConnection = jobManagerTable.get(jobId);
+
+ return jmConnection != null && jmConnection.getLeaderId().equals(leaderId);
+ }
+
// ------------------------------------------------------------------------
// Properties
// ------------------------------------------------------------------------
@@ -737,11 +969,6 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
return resourceManagerConnection;
}
- @VisibleForTesting
- public void addUnconfirmedFreeSlotNotification(SlotID slotID) {
- unconfirmedFreeSlots.add(slotID);
- }
-
// ------------------------------------------------------------------------
// Utility classes
// ------------------------------------------------------------------------
@@ -767,6 +994,44 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
+ private final class JobLeaderListenerImpl implements JobLeaderListener {
+
+ @Override
+ public void jobManagerGainedLeadership(
+ final JobID jobId,
+ final JobMasterGateway jobManagerGateway,
+ final UUID jobLeaderId,
+ final JMTMRegistrationSuccess registrationMessage) {
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ establishJobManagerConnection(
+ jobId,
+ jobManagerGateway,
+ jobLeaderId,
+ registrationMessage);
+ }
+ });
+ }
+
+ @Override
+ public void jobManagerLostLeadership(final JobID jobId, final UUID jobLeaderId) {
+ log.info("JobManager for job {} with leader id {} lost leadership.", jobId, jobLeaderId);
+
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ closeJobManagerConnection(jobId);
+ }
+ });
+ }
+
+ @Override
+ public void handleError(Throwable throwable) {
+ onFatalErrorAsync(throwable);
+ }
+ }
+
private final class TaskManagerActionsImpl implements TaskManagerActions {
private final UUID jobMasterLeaderId;
private final JobMasterGateway jobMasterGateway;
@@ -830,5 +1095,4 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
});
}
}
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
index f062b96..1ffc407 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
@@ -18,10 +18,10 @@
package org.apache.flink.runtime.taskexecutor;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.SlotID;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
@@ -30,9 +30,9 @@ import org.apache.flink.runtime.executiongraph.PartitionInfo;
import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.rpc.RpcGateway;
import org.apache.flink.runtime.rpc.RpcTimeout;
+import org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException;
import org.apache.flink.runtime.taskmanager.Task;
-import java.util.Collection;
import java.util.UUID;
/**
@@ -43,28 +43,31 @@ public interface TaskExecutorGateway extends RpcGateway {
/**
* Requests a slot from the TaskManager
*
- * @param slotID slot id for the request
- * @param allocationID id for the request
- * @param resourceManagerLeaderID current leader id of the ResourceManager
+ * @param slotId slot id for the request
+ * @param allocationId id for the request
+ * @param resourceManagerLeaderId current leader id of the ResourceManager
+ * @throws SlotAllocationException if the slot allocation fails
* @return answer to the slot request
*/
Future<TMSlotRequestReply> requestSlot(
- SlotID slotID,
- AllocationID allocationID,
- UUID resourceManagerLeaderID,
+ SlotID slotId,
+ JobID jobId,
+ AllocationID allocationId,
+ String targetAddress,
+ UUID resourceManagerLeaderId,
@RpcTimeout Time timeout);
/**
* Submit a {@link Task} to the {@link TaskExecutor}.
*
* @param tdd describing the task to submit
- * @param jobManagerID identifying the submitting JobManager
+ * @param leaderId of the job leader
* @param timeout of the submit operation
* @return Future acknowledge of the successful operation
*/
Future<Acknowledge> submitTask(
TaskDeploymentDescriptor tdd,
- ResourceID jobManagerID,
+ UUID leaderId,
@RpcTimeout Time timeout);
/**
@@ -74,7 +77,7 @@ public interface TaskExecutorGateway extends RpcGateway {
* @param partitionInfos telling where the partition can be retrieved from
* @return Future acknowledge if the partitions have been successfully updated
*/
- Future<Acknowledge> updatePartitions(ExecutionAttemptID executionAttemptID, Collection<PartitionInfo> partitionInfos);
+ Future<Acknowledge> updatePartitions(ExecutionAttemptID executionAttemptID, Iterable<PartitionInfo> partitionInfos);
/**
* Fail all intermediate result partitions of the given task.
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java
index 2dbd9eb..53f030e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java
@@ -68,11 +68,16 @@ public class TaskExecutorToResourceManagerConnection
@Override
protected void onRegistrationSuccess(TaskExecutorRegistrationSuccess success) {
+ log.info("Successful registration at resource manager {} under registration id {}.",
+ getTargetAddress(), success.getRegistrationId());
+
registrationId = success.getRegistrationId();
}
@Override
protected void onRegistrationFailure(Throwable failure) {
+ log.info("Failed to register at resource manager {}.", getTargetAddress(), failure);
+
taskExecutor.onFatalErrorAsync(failure);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
index ca1d2ce..9f78682 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
@@ -99,6 +99,8 @@ public class TaskManagerRunner implements FatalErrorHandler {
taskManagerServices.getBroadcastVariableManager(),
taskManagerServices.getFileCache(),
taskManagerServices.getTaskSlotTable(),
+ taskManagerServices.getJobManagerTable(),
+ taskManagerServices.getJobLeaderService(),
this);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
index c1728b4..7575ba3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
@@ -73,6 +73,8 @@ public class TaskManagerServices {
private final BroadcastVariableManager broadcastVariableManager;
private final FileCache fileCache;
private final TaskSlotTable taskSlotTable;
+ private final JobManagerTable jobManagerTable;
+ private final JobLeaderService jobLeaderService;
private TaskManagerServices(
TaskManagerLocation taskManagerLocation,
@@ -83,7 +85,9 @@ public class TaskManagerServices {
TaskManagerMetricGroup taskManagerMetricGroup,
BroadcastVariableManager broadcastVariableManager,
FileCache fileCache,
- TaskSlotTable taskSlotTable) {
+ TaskSlotTable taskSlotTable,
+ JobManagerTable jobManagerTable,
+ JobLeaderService jobLeaderService) {
this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
this.memoryManager = Preconditions.checkNotNull(memoryManager);
@@ -94,6 +98,8 @@ public class TaskManagerServices {
this.broadcastVariableManager = Preconditions.checkNotNull(broadcastVariableManager);
this.fileCache = Preconditions.checkNotNull(fileCache);
this.taskSlotTable = Preconditions.checkNotNull(taskSlotTable);
+ this.jobManagerTable = Preconditions.checkNotNull(jobManagerTable);
+ this.jobLeaderService = Preconditions.checkNotNull(jobLeaderService);
}
// --------------------------------------------------------------------------------------------
@@ -136,6 +142,14 @@ public class TaskManagerServices {
return taskSlotTable;
}
+ public JobManagerTable getJobManagerTable() {
+ return jobManagerTable;
+ }
+
+ public JobLeaderService getJobLeaderService() {
+ return jobLeaderService;
+ }
+
// --------------------------------------------------------------------------------------------
// Static factory methods for task manager services
// --------------------------------------------------------------------------------------------
@@ -190,6 +204,10 @@ public class TaskManagerServices {
final TimerService<AllocationID> timerService = new TimerService<>(new ScheduledThreadPoolExecutor(1));
final TaskSlotTable taskSlotTable = new TaskSlotTable(resourceProfiles, timerService);
+
+ final JobManagerTable jobManagerTable = new JobManagerTable();
+
+ final JobLeaderService jobLeaderService = new JobLeaderService(resourceID);
return new TaskManagerServices(
taskManagerLocation,
@@ -200,7 +218,9 @@ public class TaskManagerServices {
taskManagerMetricGroup,
broadcastVariableManager,
fileCache,
- taskSlotTable);
+ taskSlotTable,
+ jobManagerTable,
+ jobLeaderService);
}
/**
[26/50] [abbrv] flink git commit: [FLINK-4735] [cluster management]
Implements some job execution related RPC calls on the JobManager
Posted by tr...@apache.org.
[FLINK-4735] [cluster management] Implements some job execution related RPC calls on the JobManager
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/94a0646c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/94a0646c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/94a0646c
Branch: refs/heads/flip-6
Commit: 94a0646cbbf86f3aacd979570c979c0401004be2
Parents: bf4d384
Author: Kurt Young <yk...@gmail.com>
Authored: Tue Oct 4 23:00:22 2016 +0800
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:31 2016 +0100
----------------------------------------------------------------------
.../flink/runtime/jobmaster/JobMaster.java | 246 +++++++++++++++++--
.../runtime/jobmaster/JobMasterGateway.java | 93 ++++++-
.../jobmaster/message/ClassloadingProps.java | 68 +++++
.../message/DisposeSavepointResponse.java | 49 ++++
.../message/TriggerSavepointResponse.java | 74 ++++++
5 files changed, 507 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/94a0646c/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 8f3a342..3b8fc97 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -28,6 +28,7 @@ import org.apache.flink.core.io.InputSplit;
import org.apache.flink.core.io.InputSplitAssigner;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.checkpoint.CheckpointCoordinator;
import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
import org.apache.flink.runtime.checkpoint.CompletedCheckpointStore;
@@ -39,8 +40,11 @@ import org.apache.flink.runtime.client.JobExecutionException;
import org.apache.flink.runtime.client.JobSubmissionException;
import org.apache.flink.runtime.client.SerializedJobExecutionResult;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.execution.ExecutionState;
+import org.apache.flink.runtime.concurrent.BiFunction;
import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.concurrent.impl.FlinkFuture;
+import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager;
import org.apache.flink.runtime.executiongraph.Execution;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
@@ -61,10 +65,20 @@ import org.apache.flink.runtime.jobgraph.jsonplan.JsonPlanGenerator;
import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
import org.apache.flink.runtime.jobmanager.OnCompletionActions;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
+import org.apache.flink.runtime.jobmaster.message.ClassloadingProps;
+import org.apache.flink.runtime.jobmaster.message.DisposeSavepointResponse;
+import org.apache.flink.runtime.jobmaster.message.NextInputSplit;
+import org.apache.flink.runtime.jobmaster.message.TriggerSavepointResponse;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
-import org.apache.flink.runtime.messages.Acknowledge;
+import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
+import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint;
import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup;
+import org.apache.flink.runtime.query.KvStateID;
+import org.apache.flink.runtime.query.KvStateLocation;
+import org.apache.flink.runtime.query.KvStateLocationRegistry;
+import org.apache.flink.runtime.query.KvStateServerAddress;
+import org.apache.flink.runtime.query.UnknownKvStateLocation;
import org.apache.flink.runtime.registration.RegisteredRpcConnection;
import org.apache.flink.runtime.registration.RegistrationResponse;
import org.apache.flink.runtime.registration.RetryingRegistration;
@@ -72,7 +86,7 @@ import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.runtime.state.CheckpointStateHandles;
+import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import org.apache.flink.runtime.util.SerializedThrowable;
import org.apache.flink.util.InstantiationUtil;
@@ -520,22 +534,6 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
throw new ExecutionGraphException("The execution attempt " +
taskExecutionState.getID() + " was not found.");
}
-
- }
-
- //----------------------------------------------------------------------------------------------\u2028
- // Internal methods\u2028
- // ----------------------------------------------------------------------------------------------\u2028\u2028
-
- private void handleFatalError(final Throwable cause) {
- runAsync(new Runnable() {
- @Override
- public void run() {
- log.error("Fatal error occurred on JobManager, cause: {}", cause.getMessage(), cause);
- shutDown();
- jobCompletionActions.onFatalError(cause);
- }
- });
}
@RpcMethod
@@ -631,10 +629,220 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
throw new UnsupportedOperationException();
}
+ @RpcMethod
+ public void resourceRemoved(final ResourceID resourceId, final String message) {
+ // TODO: remove resource from slot pool
+ }
+
+ @RpcMethod
+ public void acknowledgeCheckpoint(final AcknowledgeCheckpoint acknowledge) {
+ if (executionGraph != null) {
+ final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
+ if (checkpointCoordinator != null) {
+ getRpcService().execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ if (!checkpointCoordinator.receiveAcknowledgeMessage(acknowledge)) {
+ log.info("Received message for non-existing checkpoint {}.",
+ acknowledge.getCheckpointId());
+ }
+ } catch (Exception e) {
+ log.error("Error in CheckpointCoordinator while processing {}", acknowledge, e);
+ }
+ }
+ });
+ }
+ else {
+ log.error("Received AcknowledgeCheckpoint message for job {} with no CheckpointCoordinator",
+ jobGraph.getJobID());
+ }
+ } else {
+ log.error("Received AcknowledgeCheckpoint for unavailable job {}", jobGraph.getJobID());
+ }
+ }
+
+ @RpcMethod
+ public void declineCheckpoint(final DeclineCheckpoint decline) {
+ if (executionGraph != null) {
+ final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
+ if (checkpointCoordinator != null) {
+ getRpcService().execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ if (!checkpointCoordinator.receiveDeclineMessage(decline)) {
+ log.info("Received message for non-existing checkpoint {}.", decline.getCheckpointId());
+ }
+ } catch (Exception e) {
+ log.error("Error in CheckpointCoordinator while processing {}", decline, e);
+ }
+ }
+ });
+ } else {
+ log.error("Received DeclineCheckpoint message for job {} with no CheckpointCoordinator",
+ jobGraph.getJobID());
+ }
+ } else {
+ log.error("Received AcknowledgeCheckpoint for unavailable job {}", jobGraph.getJobID());
+ }
+ }
+
+ @RpcMethod
+ public KvStateLocation lookupKvStateLocation(final String registrationName) throws Exception {
+ if (executionGraph != null) {
+ if (log.isDebugEnabled()) {
+ log.debug("Lookup key-value state for job {} with registration " +
+ "name {}.", jobGraph.getJobID(), registrationName);
+ }
+
+ final KvStateLocationRegistry registry = executionGraph.getKvStateLocationRegistry();
+ final KvStateLocation location = registry.getKvStateLocation(registrationName);
+ if (location != null) {
+ return location;
+ } else {
+ throw new UnknownKvStateLocation(registrationName);
+ }
+ } else {
+ throw new IllegalStateException("Received lookup KvState location request for unavailable job " +
+ jobGraph.getJobID());
+ }
+ }
+
+ @RpcMethod
+ public void notifyKvStateRegistered(
+ final JobVertexID jobVertexId,
+ final KeyGroupRange keyGroupRange,
+ final String registrationName,
+ final KvStateID kvStateId,
+ final KvStateServerAddress kvStateServerAddress)
+ {
+ if (executionGraph != null) {
+ if (log.isDebugEnabled()) {
+ log.debug("Key value state registered for job {} under name {}.",
+ jobGraph.getJobID(), registrationName);
+ }
+ try {
+ executionGraph.getKvStateLocationRegistry().notifyKvStateRegistered(
+ jobVertexId, keyGroupRange, registrationName, kvStateId, kvStateServerAddress
+ );
+ } catch (Exception e) {
+ log.error("Failed to notify KvStateRegistry about registration {}.", registrationName);
+ }
+ } else {
+ log.error("Received notify KvState registered request for unavailable job " + jobGraph.getJobID());
+ }
+ }
+
+ @RpcMethod
+ public void notifyKvStateUnregistered(
+ JobVertexID jobVertexId,
+ KeyGroupRange keyGroupRange,
+ String registrationName)
+ {
+ if (executionGraph != null) {
+ if (log.isDebugEnabled()) {
+ log.debug("Key value state unregistered for job {} under name {}.",
+ jobGraph.getJobID(), registrationName);
+ }
+ try {
+ executionGraph.getKvStateLocationRegistry().notifyKvStateUnregistered(
+ jobVertexId, keyGroupRange, registrationName
+ );
+ } catch (Exception e) {
+ log.error("Failed to notify KvStateRegistry about registration {}.", registrationName);
+ }
+ } else {
+ log.error("Received notify KvState unregistered request for unavailable job " + jobGraph.getJobID());
+ }
+ }
+
+ @RpcMethod
+ public Future<TriggerSavepointResponse> triggerSavepoint() throws Exception {
+ if (executionGraph != null) {
+ final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
+ if (checkpointCoordinator != null) {
+ try {
+ Future<String> savepointFuture = new FlinkFuture<>(
+ checkpointCoordinator.triggerSavepoint(System.currentTimeMillis()));
+
+ return savepointFuture.handleAsync(new BiFunction<String, Throwable, TriggerSavepointResponse>() {
+ @Override
+ public TriggerSavepointResponse apply(String savepointPath, Throwable throwable) {
+ if (throwable == null) {
+ return new TriggerSavepointResponse.Success(jobGraph.getJobID(), savepointPath);
+ }
+ else {
+ return new TriggerSavepointResponse.Failure(jobGraph.getJobID(),
+ new Exception("Failed to complete savepoint", throwable));
+ }
+ }
+ }, getMainThreadExecutor());
+
+ } catch (Exception e) {
+ FlinkCompletableFuture<TriggerSavepointResponse> future = new FlinkCompletableFuture<>();
+ future.complete(new TriggerSavepointResponse.Failure(jobGraph.getJobID(),
+ new Exception("Failed to trigger savepoint", e)));
+ return future;
+ }
+ } else {
+ FlinkCompletableFuture<TriggerSavepointResponse> future = new FlinkCompletableFuture<>();
+ future.complete(new TriggerSavepointResponse.Failure(jobGraph.getJobID(),
+ new IllegalStateException("Checkpointing disabled. You can enable it via the execution " +
+ "environment of your job.")));
+ return future;
+ }
+ } else {
+ FlinkCompletableFuture<TriggerSavepointResponse> future = new FlinkCompletableFuture<>();
+ future.complete(new TriggerSavepointResponse.Failure(jobGraph.getJobID(),
+ new IllegalArgumentException("Received trigger savepoint request for unavailable job " +
+ jobGraph.getJobID())));
+ return future;
+ }
+ }
+
+ @RpcMethod
+ public DisposeSavepointResponse disposeSavepoint(final String savepointPath) {
+ try {
+ log.info("Disposing savepoint at {}.", savepointPath);
+
+ // check whether the savepoint exists
+ savepointStore.loadSavepoint(savepointPath);
+
+ savepointStore.disposeSavepoint(savepointPath);
+ return new DisposeSavepointResponse.Success();
+ } catch (Exception e) {
+ log.error("Failed to dispose savepoint at {}.", savepointPath, e);
+ return new DisposeSavepointResponse.Failure(e);
+ }
+ }
+
+ @RpcMethod
+ public ClassloadingProps requestClassloadingProps() throws Exception {
+ if (executionGraph != null) {
+ return new ClassloadingProps(libraryCacheManager.getBlobServerPort(),
+ executionGraph.getRequiredJarFiles(),
+ executionGraph.getRequiredClasspaths());
+ } else {
+ throw new Exception("Received classloading props request for unavailable job " + jobGraph.getJobID());
+ }
+ }
+
//----------------------------------------------------------------------------------------------
// Internal methods
//----------------------------------------------------------------------------------------------
+ private void handleFatalError(final Throwable cause) {
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ log.error("Fatal error occurred on JobManager, cause: {}", cause.getMessage(), cause);
+ shutDown();
+ jobCompletionActions.onFatalError(cause);
+ }
+ });
+ }
+
// TODO - wrap this as StatusListenerMessenger's callback with rpc main thread
private void jobStatusChanged(final JobStatus newJobStatus, long timestamp, final Throwable error) {
final JobID jobID = executionGraph.getJobID();
http://git-wip-us.apache.org/repos/asf/flink/blob/94a0646c/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
index e3e57d4..4b51258 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
@@ -18,8 +18,6 @@
package org.apache.flink.runtime.jobmaster;
-import org.apache.flink.api.common.time.Time;
-import org.apache.flink.runtime.checkpoint.CheckpointCoordinatorGateway;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
@@ -30,8 +28,18 @@ import org.apache.flink.runtime.io.network.partition.ResultPartition;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.messages.Acknowledge;
-import org.apache.flink.runtime.rpc.RpcTimeout;
+import org.apache.flink.runtime.jobmaster.message.ClassloadingProps;
+import org.apache.flink.runtime.jobmaster.message.DisposeSavepointResponse;
+import org.apache.flink.runtime.jobmaster.message.NextInputSplit;
+import org.apache.flink.runtime.jobmaster.message.TriggerSavepointResponse;
+import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
+import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint;
+import org.apache.flink.runtime.query.KvStateID;
+import org.apache.flink.runtime.query.KvStateLocation;
+import org.apache.flink.runtime.query.KvStateServerAddress;
+import org.apache.flink.runtime.rpc.RpcGateway;
+import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.state.KvState;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import java.util.UUID;
@@ -110,4 +118,81 @@ public interface JobMasterGateway extends CheckpointCoordinatorGateway {
* @param resourceID identifying the TaskManager to disconnect
*/
void disconnectTaskManager(ResourceID resourceID);
+ void scheduleOrUpdateConsumers(final ResultPartitionID partitionID);
+
+ /**
+ * Notifies the JobManager about the removal of a resource.
+ *
+ * @param resourceId The ID under which the resource is registered.
+ * @param message Optional message with details, for logging and debugging.
+ */
+
+ void resourceRemoved(final ResourceID resourceId, final String message);
+
+ /**
+ * Notifies the JobManager that the checkpoint of an individual task is completed.
+ *
+ * @param acknowledge The acknowledge message of the checkpoint
+ */
+ void acknowledgeCheckpoint(final AcknowledgeCheckpoint acknowledge);
+
+ /**
+ * Notifies the JobManager that a checkpoint request could not be heeded.
+ * This can happen if a Task is already in RUNNING state but is internally not yet ready to perform checkpoints.
+ *
+ * @param decline The decline message of the checkpoint
+ */
+ void declineCheckpoint(final DeclineCheckpoint decline);
+
+ /**
+ * Requests a {@link KvStateLocation} for the specified {@link KvState} registration name.
+ *
+ * @param registrationName Name under which the KvState has been registered.
+ * @return Future of the requested {@link KvState} location
+ */
+ Future<KvStateLocation> lookupKvStateLocation(final String registrationName) throws Exception;
+
+ /**
+ * @param jobVertexId JobVertexID the KvState instance belongs to.
+ * @param keyGroupRange Key group range the KvState instance belongs to.
+ * @param registrationName Name under which the KvState has been registered.
+ * @param kvStateId ID of the registered KvState instance.
+ * @param kvStateServerAddress Server address where to find the KvState instance.
+ */
+ void notifyKvStateRegistered(
+ final JobVertexID jobVertexId,
+ final KeyGroupRange keyGroupRange,
+ final String registrationName,
+ final KvStateID kvStateId,
+ final KvStateServerAddress kvStateServerAddress);
+
+ /**
+ * @param jobVertexId JobVertexID the KvState instance belongs to.
+ * @param keyGroupRange Key group index the KvState instance belongs to.
+ * @param registrationName Name under which the KvState has been registered.
+ */
+ void notifyKvStateUnregistered(
+ JobVertexID jobVertexId,
+ KeyGroupRange keyGroupRange,
+ String registrationName);
+
+ /**
+ * Notifies the JobManager to trigger a savepoint for this job.
+ *
+ * @return Future of the savepoint trigger response.
+ */
+ Future<TriggerSavepointResponse> triggerSavepoint();
+
+ /**
+ * Notifies the Jobmanager to dispose specified savepoint.
+ *
+ * @param savepointPath The path of the savepoint.
+ * @return The future of the savepoint disponse response.
+ */
+ Future<DisposeSavepointResponse> disposeSavepoint(final String savepointPath);
+
+ /**
+ * Request the classloading props of this job.
+ */
+ Future<ClassloadingProps> requestClassloadingProps();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/94a0646c/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/ClassloadingProps.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/ClassloadingProps.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/ClassloadingProps.java
new file mode 100644
index 0000000..2d670b4
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/ClassloadingProps.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster.message;
+
+import org.apache.flink.runtime.blob.BlobKey;
+
+import java.io.Serializable;
+import java.net.URL;
+import java.util.List;
+
+/**
+ * The response of classloading props request to JobManager.
+ */
+public class ClassloadingProps implements Serializable {
+
+ private static final long serialVersionUID = -3282341310808511823L;
+
+ private final int blobManagerPort;
+
+ private final List<BlobKey> requiredJarFiles;
+
+ private final List<URL> requiredClasspaths;
+
+ /**
+ * Constructor of ClassloadingProps.
+ *
+ * @param blobManagerPort The port of the blobManager
+ * @param requiredJarFiles The blob keys of the required jar files
+ * @param requiredClasspaths The urls of the required classpaths
+ */
+ public ClassloadingProps(
+ final int blobManagerPort,
+ final List<BlobKey> requiredJarFiles,
+ final List<URL> requiredClasspaths)
+ {
+ this.blobManagerPort = blobManagerPort;
+ this.requiredJarFiles = requiredJarFiles;
+ this.requiredClasspaths = requiredClasspaths;
+ }
+
+ public int getBlobManagerPort() {
+ return blobManagerPort;
+ }
+
+ public List<BlobKey> getRequiredJarFiles() {
+ return requiredJarFiles;
+ }
+
+ public List<URL> getRequiredClasspaths() {
+ return requiredClasspaths;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/94a0646c/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/DisposeSavepointResponse.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/DisposeSavepointResponse.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/DisposeSavepointResponse.java
new file mode 100644
index 0000000..42bfc71
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/DisposeSavepointResponse.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster.message;
+
+import java.io.Serializable;
+
+/**
+ * The response of the dispose savepoint request to JobManager.
+ */
+public abstract class DisposeSavepointResponse implements Serializable {
+
+ private static final long serialVersionUID = 6008792963949369567L;
+
+ public static class Success extends DisposeSavepointResponse implements Serializable {
+
+ private static final long serialVersionUID = 1572462960008711415L;
+ }
+
+ public static class Failure extends DisposeSavepointResponse implements Serializable {
+
+ private static final long serialVersionUID = -7505308325483022458L;
+
+ private final Throwable cause;
+
+ public Failure(final Throwable cause) {
+ this.cause = cause;
+ }
+
+ public Throwable getCause() {
+ return cause;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/94a0646c/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/TriggerSavepointResponse.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/TriggerSavepointResponse.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/TriggerSavepointResponse.java
new file mode 100644
index 0000000..0b0edc5
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/TriggerSavepointResponse.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster.message;
+
+import org.apache.flink.api.common.JobID;
+
+import java.io.Serializable;
+
+/**
+ * The response of the trigger savepoint request to JobManager.
+ */
+public abstract class TriggerSavepointResponse implements Serializable {
+
+ private static final long serialVersionUID = 3139327824611807707L;
+
+ private final JobID jobID;
+
+ public JobID getJobID() {
+ return jobID;
+ }
+
+ public TriggerSavepointResponse(final JobID jobID) {
+ this.jobID = jobID;
+ }
+
+ public static class Success extends TriggerSavepointResponse implements Serializable {
+
+ private static final long serialVersionUID = -1100637460388881776L;
+
+ private final String savepointPath;
+
+ public Success(final JobID jobID, final String savepointPath) {
+ super(jobID);
+ this.savepointPath = savepointPath;
+ }
+
+ public String getSavepointPath() {
+ return savepointPath;
+ }
+ }
+
+ public static class Failure extends TriggerSavepointResponse implements Serializable {
+
+ private static final long serialVersionUID = -1668479003490615139L;
+
+ private final Throwable cause;
+
+ public Failure(final JobID jobID, final Throwable cause) {
+ super(jobID);
+ this.cause = cause;
+ }
+
+ public Throwable getCause() {
+ return cause;
+ }
+ }
+}
+
[02/50] [abbrv] flink git commit: [FLINK-4505] [cluster mngt]
Implement TaskManager component's startup
Posted by tr...@apache.org.
[FLINK-4505] [cluster mngt] Implement TaskManager component's startup
The TaskManagerRunner now contains the startup logic for the TaskManager's components.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/0a32c6c1
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/0a32c6c1
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/0a32c6c1
Branch: refs/heads/flip-6
Commit: 0a32c6c18f78f528a33b5d42cd7493cc10288a65
Parents: 20e4e12
Author: \u6dd8\u6c5f <ta...@alibaba-inc.com>
Authored: Fri Sep 2 18:00:49 2016 +0800
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:27 2016 +0100
----------------------------------------------------------------------
.../runtime/taskexecutor/TaskExecutor.java | 766 +------------------
.../runtime/taskmanager/TaskManagerRunner.java | 749 ++++++++++++++++++
.../runtime/taskexecutor/TaskExecutorTest.java | 53 +-
3 files changed, 804 insertions(+), 764 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/0a32c6c1/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index 9d9ad2a..8ce2780 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -18,74 +18,29 @@
package org.apache.flink.runtime.taskexecutor;
-import akka.actor.ActorSystem;
-import com.typesafe.config.Config;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
-import org.apache.flink.runtime.io.network.ConnectionManager;
-import org.apache.flink.runtime.io.network.LocalConnectionManager;
-import org.apache.flink.runtime.io.network.TaskEventDispatcher;
-import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
-import org.apache.flink.runtime.io.network.netty.NettyConnectionManager;
-import org.apache.flink.runtime.io.network.partition.ResultPartitionManager;
import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
-import org.apache.flink.runtime.query.KvStateRegistry;
-import org.apache.flink.runtime.query.netty.DisabledKvStateRequestStats;
-import org.apache.flink.runtime.query.netty.KvStateServer;
import org.apache.flink.runtime.resourcemanager.SlotRequestRegistered;
import org.apache.flink.runtime.resourcemanager.SlotRequestReply;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
-import org.apache.flink.util.Preconditions;
import org.jboss.netty.channel.ChannelException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.IllegalConfigurationException;
-import org.apache.flink.core.memory.HeapMemorySegment;
-import org.apache.flink.core.memory.HybridMemorySegment;
-import org.apache.flink.core.memory.MemorySegmentFactory;
-import org.apache.flink.core.memory.MemoryType;
-import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
-import org.apache.flink.runtime.io.disk.iomanager.IOManager.IOMode;
-import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync;
import org.apache.flink.runtime.io.network.NetworkEnvironment;
-import org.apache.flink.runtime.io.network.netty.NettyConfig;
-import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.memory.MemoryManager;
import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
-import org.apache.flink.runtime.taskmanager.MemoryLogger;
-import org.apache.flink.runtime.util.EnvironmentInformation;
-import org.apache.flink.runtime.util.LeaderRetrievalUtils;
-import org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration;
-import org.apache.flink.util.MathUtils;
-import org.apache.flink.util.NetUtils;
-
-import scala.Tuple2;
-import scala.Option;
-import scala.Some;
-import scala.concurrent.duration.Duration;
-import scala.concurrent.duration.FiniteDuration;
-
-import java.io.File;
-import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.net.BindException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
import java.util.UUID;
-import java.util.concurrent.TimeUnit;
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -98,12 +53,10 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
private static final Logger LOG = LoggerFactory.getLogger(TaskExecutor.class);
- /** The unique resource ID of this TaskExecutor */
- private final ResourceID resourceID;
-
+ /** The connection information of this task manager */
private final TaskManagerLocation taskManagerLocation;
- /** The access to the leader election and metadata storage services */
+ /** The access to the leader election and retrieval services */
private final HighAvailabilityServices haServices;
/** The task manager configuration */
@@ -128,28 +81,26 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
// ------------------------------------------------------------------------
public TaskExecutor(
- TaskExecutorConfiguration taskExecutorConfig,
- ResourceID resourceID,
- TaskManagerLocation taskManagerLocation,
- MemoryManager memoryManager,
- IOManager ioManager,
- NetworkEnvironment networkEnvironment,
- RpcService rpcService,
- HighAvailabilityServices haServices) {
+ TaskExecutorConfiguration taskExecutorConfig,
+ TaskManagerLocation taskManagerLocation,
+ RpcService rpcService,
+ MemoryManager memoryManager,
+ IOManager ioManager,
+ NetworkEnvironment networkEnvironment,
+ HighAvailabilityServices haServices) {
super(rpcService);
checkArgument(taskExecutorConfig.getNumberOfSlots() > 0, "The number of slots has to be larger than 0.");
this.taskExecutorConfig = checkNotNull(taskExecutorConfig);
- this.resourceID = checkNotNull(resourceID);
this.taskManagerLocation = checkNotNull(taskManagerLocation);
this.memoryManager = checkNotNull(memoryManager);
this.ioManager = checkNotNull(ioManager);
this.networkEnvironment = checkNotNull(networkEnvironment);
this.haServices = checkNotNull(haServices);
- this.numberOfSlots = taskExecutorConfig.getNumberOfSlots();
+ this.numberOfSlots = taskExecutorConfig.getNumberOfSlots();
}
// ------------------------------------------------------------------------
@@ -207,7 +158,6 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
- /**
* Requests a slot from the TaskManager
*
* @param allocationID id for the request
@@ -220,126 +170,11 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
/**
- * Starts and runs the TaskManager.
- * <p/>
- * This method first tries to select the network interface to use for the TaskManager
- * communication. The network interface is used both for the actor communication
- * (coordination) as well as for the data exchange between task managers. Unless
- * the hostname/interface is explicitly configured in the configuration, this
- * method will try out various interfaces and methods to connect to the JobManager
- * and select the one where the connection attempt is successful.
- * <p/>
- * After selecting the network interface, this method brings up an actor system
- * for the TaskManager and its actors, starts the TaskManager's services
- * (library cache, shuffle network stack, ...), and starts the TaskManager itself.
- *
- * @param configuration The configuration for the TaskManager.
- * @param resourceID The id of the resource which the task manager will run on.
- */
- public static void selectNetworkInterfaceAndRunTaskManager(
- Configuration configuration,
- ResourceID resourceID) throws Exception {
-
- final InetSocketAddress taskManagerAddress = selectNetworkInterfaceAndPort(configuration);
-
- runTaskManager(taskManagerAddress.getHostName(), resourceID, taskManagerAddress.getPort(), configuration);
- }
-
- private static InetSocketAddress selectNetworkInterfaceAndPort(Configuration configuration)
- throws Exception {
- String taskManagerHostname = configuration.getString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, null);
- if (taskManagerHostname != null) {
- LOG.info("Using configured hostname/address for TaskManager: " + taskManagerHostname);
- } else {
- LeaderRetrievalService leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(configuration);
- FiniteDuration lookupTimeout = AkkaUtils.getLookupTimeout(configuration);
-
- InetAddress taskManagerAddress = LeaderRetrievalUtils.findConnectingAddress(leaderRetrievalService, lookupTimeout);
- taskManagerHostname = taskManagerAddress.getHostName();
- LOG.info("TaskManager will use hostname/address '{}' ({}) for communication.",
- taskManagerHostname, taskManagerAddress.getHostAddress());
- }
-
- // if no task manager port has been configured, use 0 (system will pick any free port)
- final int actorSystemPort = configuration.getInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, 0);
- if (actorSystemPort < 0 || actorSystemPort > 65535) {
- throw new IllegalConfigurationException("Invalid value for '" +
- ConfigConstants.TASK_MANAGER_IPC_PORT_KEY +
- "' (port for the TaskManager actor system) : " + actorSystemPort +
- " - Leave config parameter empty or use 0 to let the system choose a port automatically.");
- }
-
- return new InetSocketAddress(taskManagerHostname, actorSystemPort);
- }
-
- /**
- * Starts and runs the TaskManager. Brings up an actor system for the TaskManager and its
- * actors, starts the TaskManager's services (library cache, shuffle network stack, ...),
- * and starts the TaskManager itself.
- * <p/>
- * This method will also spawn a process reaper for the TaskManager (kill the process if
- * the actor fails) and optionally start the JVM memory logging thread.
- *
- * @param taskManagerHostname The hostname/address of the interface where the actor system
- * will communicate.
- * @param resourceID The id of the resource which the task manager will run on.
- * @param actorSystemPort The port at which the actor system will communicate.
- * @param configuration The configuration for the TaskManager.
- */
- private static void runTaskManager(
- String taskManagerHostname,
- ResourceID resourceID,
- int actorSystemPort,
- final Configuration configuration) throws Exception {
-
- LOG.info("Starting TaskManager");
-
- // Bring up the TaskManager actor system first, bind it to the given address.
-
- LOG.info("Starting TaskManager actor system at " +
- NetUtils.hostAndPortToUrlString(taskManagerHostname, actorSystemPort));
-
- final ActorSystem taskManagerSystem;
- try {
- Tuple2<String, Object> address = new Tuple2<String, Object>(taskManagerHostname, actorSystemPort);
- Config akkaConfig = AkkaUtils.getAkkaConfig(configuration, new Some<>(address));
- LOG.debug("Using akka configuration\n " + akkaConfig);
- taskManagerSystem = AkkaUtils.createActorSystem(akkaConfig);
- } catch (Throwable t) {
- if (t instanceof ChannelException) {
- Throwable cause = t.getCause();
- if (cause != null && t.getCause() instanceof BindException) {
- String address = NetUtils.hostAndPortToUrlString(taskManagerHostname, actorSystemPort);
- throw new IOException("Unable to bind TaskManager actor system to address " +
- address + " - " + cause.getMessage(), t);
- }
- }
- throw new Exception("Could not create TaskManager actor system", t);
- }
-
- // start akka rpc service based on actor system
- final Time timeout = Time.milliseconds(AkkaUtils.getTimeout(configuration).toMillis());
- final AkkaRpcService akkaRpcService = new AkkaRpcService(taskManagerSystem, timeout);
-
- // start high availability service to implement getResourceManagerLeaderRetriever method only
- final HighAvailabilityServices haServices = new HighAvailabilityServices() {
- @Override
- public LeaderRetrievalService getResourceManagerLeaderRetriever() throws Exception {
- return LeaderRetrievalUtils.createLeaderRetrievalService(configuration);
- }
-
- @Override
public LeaderRetrievalService getJobMasterLeaderRetriever(JobID jobID) throws Exception {
return null;
}
@Override
- public LeaderElectionService getResourceManagerLeaderElectionService() throws Exception {
- return null;
- }
-
- @Override
- public LeaderElectionService getJobMasterLeaderElectionService(JobID jobID) throws Exception {
return null;
}
@@ -350,552 +185,12 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
@Override
public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
- return null;
- }
- };
-
- // start all the TaskManager services (network stack, library cache, ...)
- // and the TaskManager actor
- try {
- LOG.info("Starting TaskManager actor");
- TaskExecutor taskExecutor = startTaskManagerComponentsAndActor(
- configuration,
- resourceID,
- akkaRpcService,
- taskManagerHostname,
- haServices,
- false);
-
- taskExecutor.start();
-
- // if desired, start the logging daemon that periodically logs the memory usage information
- if (LOG.isInfoEnabled() && configuration.getBoolean(
- ConfigConstants.TASK_MANAGER_DEBUG_MEMORY_USAGE_START_LOG_THREAD,
- ConfigConstants.DEFAULT_TASK_MANAGER_DEBUG_MEMORY_USAGE_START_LOG_THREAD)) {
- LOG.info("Starting periodic memory usage logger");
-
- long interval = configuration.getLong(
- ConfigConstants.TASK_MANAGER_DEBUG_MEMORY_USAGE_LOG_INTERVAL_MS,
- ConfigConstants.DEFAULT_TASK_MANAGER_DEBUG_MEMORY_USAGE_LOG_INTERVAL_MS);
-
- MemoryLogger logger = new MemoryLogger(LOG, interval, taskManagerSystem);
- logger.start();
- }
-
- // block until everything is done
- taskManagerSystem.awaitTermination();
- } catch (Throwable t) {
- LOG.error("Error while starting up taskManager", t);
- try {
- taskManagerSystem.shutdown();
- } catch (Throwable tt) {
- LOG.warn("Could not cleanly shut down actor system", tt);
- }
- throw t;
- }
- }
-
- // --------------------------------------------------------------------------
- // Starting and running the TaskManager
- // --------------------------------------------------------------------------
-
- /**
- * @param configuration The configuration for the TaskManager.
- * @param resourceID The id of the resource which the task manager will run on.
- * @param rpcService The rpc service which is used to start and connect to the TaskManager RpcEndpoint .
- * @param taskManagerHostname The hostname/address that describes the TaskManager's data location.
- * @param haServices Optionally, a high availability service can be provided. If none is given,
- * then a HighAvailabilityServices is constructed from the configuration.
- * @param localTaskManagerCommunication If true, the TaskManager will not initiate the TCP network stack.
- * @return An ActorRef to the TaskManager actor.
- * @throws IllegalConfigurationException Thrown, if the given config contains illegal values.
- * @throws IOException Thrown, if any of the I/O components (such as buffer pools,
- * I/O manager, ...) cannot be properly started.
- * @throws Exception Thrown is some other error occurs while parsing the configuration
- * or starting the TaskManager components.
- */
- public static TaskExecutor startTaskManagerComponentsAndActor(
- Configuration configuration,
- ResourceID resourceID,
- RpcService rpcService,
- String taskManagerHostname,
- HighAvailabilityServices haServices,
- boolean localTaskManagerCommunication) throws Exception {
-
- final TaskExecutorConfiguration taskExecutorConfig = parseTaskManagerConfiguration(
- configuration, taskManagerHostname, localTaskManagerCommunication);
-
- TaskManagerComponents taskManagerComponents = createTaskManagerComponents(
- resourceID,
- InetAddress.getByName(taskManagerHostname),
- taskExecutorConfig,
- configuration);
-
- final TaskExecutor taskExecutor = new TaskExecutor(
- taskExecutorConfig,
- resourceID,
- taskManagerComponents.getTaskManagerLocation(),
- taskManagerComponents.getMemoryManager(),
- taskManagerComponents.getIOManager(),
- taskManagerComponents.getNetworkEnvironment(),
- rpcService,
- haServices);
-
- return taskExecutor;
- }
-
- /**
- * Creates and returns the task manager components.
- *
- * @param resourceID resource ID of the task manager
- * @param taskManagerAddress address of the task manager
- * @param taskExecutorConfig task manager configuration
- * @param configuration of Flink
- * @return task manager components
- * @throws Exception
- */
- private static TaskExecutor.TaskManagerComponents createTaskManagerComponents(
- ResourceID resourceID,
- InetAddress taskManagerAddress,
- TaskExecutorConfiguration taskExecutorConfig,
- Configuration configuration) throws Exception {
- MemoryType memType = taskExecutorConfig.getNetworkConfig().memoryType();
-
- // pre-start checks
- checkTempDirs(taskExecutorConfig.getTmpDirPaths());
-
- NetworkEnvironmentConfiguration networkEnvironmentConfiguration = taskExecutorConfig.getNetworkConfig();
-
- NetworkBufferPool networkBufferPool = new NetworkBufferPool(
- networkEnvironmentConfiguration.numNetworkBuffers(),
- networkEnvironmentConfiguration.networkBufferSize(),
- networkEnvironmentConfiguration.memoryType());
-
- ConnectionManager connectionManager;
-
- if (networkEnvironmentConfiguration.nettyConfig().isDefined()) {
- connectionManager = new NettyConnectionManager(networkEnvironmentConfiguration.nettyConfig().get());
- } else {
- connectionManager = new LocalConnectionManager();
- }
-
- ResultPartitionManager resultPartitionManager = new ResultPartitionManager();
- TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
-
- KvStateRegistry kvStateRegistry = new KvStateRegistry();
-
- KvStateServer kvStateServer;
-
- if (networkEnvironmentConfiguration.nettyConfig().isDefined()) {
- NettyConfig nettyConfig = networkEnvironmentConfiguration.nettyConfig().get();
-
- int numNetworkThreads = networkEnvironmentConfiguration.queryServerNetworkThreads() == 0 ?
- nettyConfig.getNumberOfSlots() : networkEnvironmentConfiguration.queryServerNetworkThreads();
-
- int numQueryThreads = networkEnvironmentConfiguration.queryServerQueryThreads() == 0 ?
- nettyConfig.getNumberOfSlots() : networkEnvironmentConfiguration.queryServerQueryThreads();
-
- kvStateServer = new KvStateServer(
- taskManagerAddress,
- networkEnvironmentConfiguration.queryServerPort(),
- numNetworkThreads,
- numQueryThreads,
- kvStateRegistry,
- new DisabledKvStateRequestStats());
- } else {
- kvStateServer = null;
- }
-
- // we start the network first, to make sure it can allocate its buffers first
- final NetworkEnvironment network = new NetworkEnvironment(
- networkBufferPool,
- connectionManager,
- resultPartitionManager,
- taskEventDispatcher,
- kvStateRegistry,
- kvStateServer,
- networkEnvironmentConfiguration.ioMode(),
- networkEnvironmentConfiguration.partitionRequestInitialBackoff(),
- networkEnvironmentConfiguration.partitinRequestMaxBackoff());
-
- network.start();
-
- TaskManagerLocation taskManagerLocation = new TaskManagerLocation(
- resourceID,
- taskManagerAddress,
- network.getConnectionManager().getDataPort());
-
- // computing the amount of memory to use depends on how much memory is available
- // it strictly needs to happen AFTER the network stack has been initialized
-
- // check if a value has been configured
- long configuredMemory = configuration.getLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
- checkConfigParameter(configuredMemory == -1 || configuredMemory > 0, configuredMemory,
- ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY,
- "MemoryManager needs at least one MB of memory. " +
- "If you leave this config parameter empty, the system automatically " +
- "pick a fraction of the available memory.");
-
- final long memorySize;
- boolean preAllocateMemory = configuration.getBoolean(
- ConfigConstants.TASK_MANAGER_MEMORY_PRE_ALLOCATE_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_PRE_ALLOCATE);
- if (configuredMemory > 0) {
- if (preAllocateMemory) {
- LOG.info("Using {} MB for managed memory." , configuredMemory);
- } else {
- LOG.info("Limiting managed memory to {} MB, memory will be allocated lazily." , configuredMemory);
- }
- memorySize = configuredMemory << 20; // megabytes to bytes
- } else {
- float fraction = configuration.getFloat(
- ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY,
- ConfigConstants.DEFAULT_MEMORY_MANAGER_MEMORY_FRACTION);
- checkConfigParameter(fraction > 0.0f && fraction < 1.0f, fraction,
- ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY,
- "MemoryManager fraction of the free memory must be between 0.0 and 1.0");
-
- if (memType == MemoryType.HEAP) {
- long relativeMemSize = (long) (EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag() * fraction);
- if (preAllocateMemory) {
- LOG.info("Using {} of the currently free heap space for managed heap memory ({} MB)." ,
- fraction , relativeMemSize >> 20);
- } else {
- LOG.info("Limiting managed memory to {} of the currently free heap space ({} MB), " +
- "memory will be allocated lazily." , fraction , relativeMemSize >> 20);
- }
- memorySize = relativeMemSize;
- } else if (memType == MemoryType.OFF_HEAP) {
- // The maximum heap memory has been adjusted according to the fraction
- long maxMemory = EnvironmentInformation.getMaxJvmHeapMemory();
- long directMemorySize = (long) (maxMemory / (1.0 - fraction) * fraction);
- if (preAllocateMemory) {
- LOG.info("Using {} of the maximum memory size for managed off-heap memory ({} MB)." ,
- fraction, directMemorySize >> 20);
- } else {
- LOG.info("Limiting managed memory to {} of the maximum memory size ({} MB)," +
- " memory will be allocated lazily.", fraction, directMemorySize >> 20);
- }
- memorySize = directMemorySize;
- } else {
- throw new RuntimeException("No supported memory type detected.");
- }
- }
-
- // now start the memory manager
- final MemoryManager memoryManager;
- try {
- memoryManager = new MemoryManager(
- memorySize,
- taskExecutorConfig.getNumberOfSlots(),
- taskExecutorConfig.getNetworkConfig().networkBufferSize(),
- memType,
- preAllocateMemory);
- } catch (OutOfMemoryError e) {
- if (memType == MemoryType.HEAP) {
- throw new Exception("OutOfMemory error (" + e.getMessage() +
- ") while allocating the TaskManager heap memory (" + memorySize + " bytes).", e);
- } else if (memType == MemoryType.OFF_HEAP) {
- throw new Exception("OutOfMemory error (" + e.getMessage() +
- ") while allocating the TaskManager off-heap memory (" + memorySize +
- " bytes).Try increasing the maximum direct memory (-XX:MaxDirectMemorySize)", e);
- } else {
- throw e;
- }
- }
-
- // start the I/O manager, it will create some temp directories.
- final IOManager ioManager = new IOManagerAsync(taskExecutorConfig.getTmpDirPaths());
-
- return new TaskExecutor.TaskManagerComponents(taskManagerLocation, memoryManager, ioManager, network);
- }
-
- // --------------------------------------------------------------------------
- // Parsing and checking the TaskManager Configuration
- // --------------------------------------------------------------------------
-
- /**
- * Utility method to extract TaskManager config parameters from the configuration and to
- * sanity check them.
- *
- * @param configuration The configuration.
- * @param taskManagerHostname The host name under which the TaskManager communicates.
- * @param localTaskManagerCommunication True, to skip initializing the network stack.
- * Use only in cases where only one task manager runs.
- * @return TaskExecutorConfiguration that wrappers InstanceConnectionInfo, NetworkEnvironmentConfiguration, etc.
- */
- private static TaskExecutorConfiguration parseTaskManagerConfiguration(
- Configuration configuration,
- String taskManagerHostname,
- boolean localTaskManagerCommunication) throws Exception {
-
- // ------- read values from the config and check them ---------
- // (a lot of them)
-
- // ----> hosts / ports for communication and data exchange
-
- int dataport = configuration.getInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_DATA_PORT);
- if (dataport == 0) {
- dataport = NetUtils.getAvailablePort();
- }
- checkConfigParameter(dataport > 0, dataport, ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
- "Leave config parameter empty or use 0 to let the system choose a port automatically.");
-
- InetAddress taskManagerAddress = InetAddress.getByName(taskManagerHostname);
- final InetSocketAddress taskManagerInetSocketAddress = new InetSocketAddress(taskManagerAddress, dataport);
-
- // ----> memory / network stack (shuffles/broadcasts), task slots, temp directories
-
- // we need this because many configs have been written with a "-1" entry
- int slots = configuration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
- if (slots == -1) {
- slots = 1;
- }
- checkConfigParameter(slots >= 1, slots, ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS,
- "Number of task slots must be at least one.");
-
- final int numNetworkBuffers = configuration.getInteger(
- ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS);
- checkConfigParameter(numNetworkBuffers > 0, numNetworkBuffers,
- ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, "");
-
- final int pageSize = configuration.getInteger(
- ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_SEGMENT_SIZE);
- // check page size of for minimum size
- checkConfigParameter(pageSize >= MemoryManager.MIN_PAGE_SIZE, pageSize,
- ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
- "Minimum memory segment size is " + MemoryManager.MIN_PAGE_SIZE);
- // check page size for power of two
- checkConfigParameter(MathUtils.isPowerOf2(pageSize), pageSize,
- ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
- "Memory segment size must be a power of 2.");
-
- // check whether we use heap or off-heap memory
- final MemoryType memType;
- if (configuration.getBoolean(ConfigConstants.TASK_MANAGER_MEMORY_OFF_HEAP_KEY, false)) {
- memType = MemoryType.OFF_HEAP;
- } else {
- memType = MemoryType.HEAP;
- }
-
- // initialize the memory segment factory accordingly
- if (memType == MemoryType.HEAP) {
- if (!MemorySegmentFactory.initializeIfNotInitialized(HeapMemorySegment.FACTORY)) {
- throw new Exception("Memory type is set to heap memory, but memory segment " +
- "factory has been initialized for off-heap memory segments");
- }
- } else {
- if (!MemorySegmentFactory.initializeIfNotInitialized(HybridMemorySegment.FACTORY)) {
- throw new Exception("Memory type is set to off-heap memory, but memory segment " +
- "factory has been initialized for heap memory segments");
- }
- }
-
- final String[] tmpDirs = configuration.getString(
- ConfigConstants.TASK_MANAGER_TMP_DIR_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH).split(",|" + File.pathSeparator);
-
- final NettyConfig nettyConfig;
- if (!localTaskManagerCommunication) {
- nettyConfig = new NettyConfig(
- taskManagerInetSocketAddress.getAddress(),
- taskManagerInetSocketAddress.getPort(),
- pageSize,
- slots,
- configuration);
- } else {
- nettyConfig = null;
- }
-
- // Default spill I/O mode for intermediate results
- final String syncOrAsync = configuration.getString(
- ConfigConstants.TASK_MANAGER_NETWORK_DEFAULT_IO_MODE,
- ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_DEFAULT_IO_MODE);
-
- final IOMode ioMode;
- if (syncOrAsync.equals("async")) {
- ioMode = IOManager.IOMode.ASYNC;
- } else {
- ioMode = IOManager.IOMode.SYNC;
- }
-
- final int queryServerPort = configuration.getInteger(
- ConfigConstants.QUERYABLE_STATE_SERVER_PORT,
- ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_PORT);
-
- final int queryServerNetworkThreads = configuration.getInteger(
- ConfigConstants.QUERYABLE_STATE_SERVER_NETWORK_THREADS,
- ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_NETWORK_THREADS);
-
- final int queryServerQueryThreads = configuration.getInteger(
- ConfigConstants.QUERYABLE_STATE_SERVER_QUERY_THREADS,
- ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_QUERY_THREADS);
-
- final NetworkEnvironmentConfiguration networkConfig = new NetworkEnvironmentConfiguration(
- numNetworkBuffers,
- pageSize,
- memType,
- ioMode,
- queryServerPort,
- queryServerNetworkThreads,
- queryServerQueryThreads,
- Option.apply(nettyConfig),
- 500,
- 30000);
-
- // ----> timeouts, library caching, profiling
-
- final FiniteDuration timeout;
- try {
- timeout = AkkaUtils.getTimeout(configuration);
- } catch (Exception e) {
- throw new IllegalArgumentException(
- "Invalid format for '" + ConfigConstants.AKKA_ASK_TIMEOUT +
- "'.Use formats like '50 s' or '1 min' to specify the timeout.");
- }
- LOG.info("Messages between TaskManager and JobManager have a max timeout of " + timeout);
-
- final long cleanupInterval = configuration.getLong(
- ConfigConstants.LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL,
- ConfigConstants.DEFAULT_LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL) * 1000;
-
- final FiniteDuration finiteRegistrationDuration;
- try {
- Duration maxRegistrationDuration = Duration.create(configuration.getString(
- ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION,
- ConfigConstants.DEFAULT_TASK_MANAGER_MAX_REGISTRATION_DURATION));
- if (maxRegistrationDuration.isFinite()) {
- finiteRegistrationDuration = new FiniteDuration(maxRegistrationDuration.toSeconds(), TimeUnit.SECONDS);
- } else {
- finiteRegistrationDuration = null;
- }
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("Invalid format for parameter " +
- ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION, e);
- }
-
- final FiniteDuration initialRegistrationPause;
- try {
- Duration pause = Duration.create(configuration.getString(
- ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE,
- ConfigConstants.DEFAULT_TASK_MANAGER_INITIAL_REGISTRATION_PAUSE));
- if (pause.isFinite()) {
- initialRegistrationPause = new FiniteDuration(pause.toSeconds(), TimeUnit.SECONDS);
- } else {
- throw new IllegalArgumentException("The initial registration pause must be finite: " + pause);
- }
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("Invalid format for parameter " +
- ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
- }
-
- final FiniteDuration maxRegistrationPause;
- try {
- Duration pause = Duration.create(configuration.getString(
- ConfigConstants.TASK_MANAGER_MAX_REGISTARTION_PAUSE,
- ConfigConstants.DEFAULT_TASK_MANAGER_MAX_REGISTRATION_PAUSE));
- if (pause.isFinite()) {
- maxRegistrationPause = new FiniteDuration(pause.toSeconds(), TimeUnit.SECONDS);
- } else {
- throw new IllegalArgumentException("The maximum registration pause must be finite: " + pause);
- }
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("Invalid format for parameter " +
- ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
- }
-
- final FiniteDuration refusedRegistrationPause;
- try {
- Duration pause = Duration.create(configuration.getString(
- ConfigConstants.TASK_MANAGER_REFUSED_REGISTRATION_PAUSE,
- ConfigConstants.DEFAULT_TASK_MANAGER_REFUSED_REGISTRATION_PAUSE));
- if (pause.isFinite()) {
- refusedRegistrationPause = new FiniteDuration(pause.toSeconds(), TimeUnit.SECONDS);
- } else {
- throw new IllegalArgumentException("The refused registration pause must be finite: " + pause);
- }
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("Invalid format for parameter " +
- ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
- }
-
- return new TaskExecutorConfiguration(
- tmpDirs,
- cleanupInterval,
- networkConfig,
- timeout,
- finiteRegistrationDuration,
- slots,
- configuration,
- initialRegistrationPause,
- maxRegistrationPause,
- refusedRegistrationPause);
- }
-
- /**
- * Validates a condition for a config parameter and displays a standard exception, if the
- * the condition does not hold.
- *
- * @param condition The condition that must hold. If the condition is false, an exception is thrown.
- * @param parameter The parameter value. Will be shown in the exception message.
- * @param name The name of the config parameter. Will be shown in the exception message.
- * @param errorMessage The optional custom error message to append to the exception message.
- */
- private static void checkConfigParameter(
- boolean condition,
- Object parameter,
- String name,
- String errorMessage) {
- if (!condition) {
- throw new IllegalConfigurationException("Invalid configuration value for " + name + " : " + parameter + " - " + errorMessage);
- }
- }
-
- /**
- * Validates that all the directories denoted by the strings do actually exist, are proper
- * directories (not files), and are writable.
- *
- * @param tmpDirs The array of directory paths to check.
- * @throws Exception Thrown if any of the directories does not exist or is not writable
- * or is a file, rather than a directory.
- */
- private static void checkTempDirs(String[] tmpDirs) throws IOException {
- for (String dir : tmpDirs) {
- if (dir != null && !dir.equals("")) {
- File file = new File(dir);
- if (!file.exists()) {
- throw new IOException("Temporary file directory " + file.getAbsolutePath() + " does not exist.");
- }
- if (!file.isDirectory()) {
- throw new IOException("Temporary file directory " + file.getAbsolutePath() + " is not a directory.");
- }
- if (!file.canWrite()) {
- throw new IOException("Temporary file directory " + file.getAbsolutePath() + " is not writable.");
- }
-
- if (LOG.isInfoEnabled()) {
- long totalSpaceGb = file.getTotalSpace() >> 30;
- long usableSpaceGb = file.getUsableSpace() >> 30;
- double usablePercentage = (double)usableSpaceGb / totalSpaceGb * 100;
- String path = file.getAbsolutePath();
- LOG.info(String.format("Temporary file directory '%s': total %d GB, " + "usable %d GB (%.2f%% usable)",
- path, totalSpaceGb, usableSpaceGb, usablePercentage));
- }
- } else {
- throw new IllegalArgumentException("Temporary file directory #$id is null.");
- }
- }
- }
-
// ------------------------------------------------------------------------
// Properties
// ------------------------------------------------------------------------
public ResourceID getResourceID() {
- return resourceID;
+ return taskManagerLocation.getResourceID();
}
// ------------------------------------------------------------------------
@@ -959,37 +254,4 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
- private static class TaskManagerComponents {
- private final TaskManagerLocation taskManagerLocation;
- private final MemoryManager memoryManager;
- private final IOManager ioManager;
- private final NetworkEnvironment networkEnvironment;
-
- private TaskManagerComponents(
- TaskManagerLocation taskManagerLocation,
- MemoryManager memoryManager,
- IOManager ioManager,
- NetworkEnvironment networkEnvironment) {
- this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
- this.memoryManager = Preconditions.checkNotNull(memoryManager);
- this.ioManager = Preconditions.checkNotNull(ioManager);
- this.networkEnvironment = Preconditions.checkNotNull(networkEnvironment);
- }
-
- public MemoryManager getMemoryManager() {
- return memoryManager;
- }
-
- public IOManager getIOManager() {
- return ioManager;
- }
-
- public NetworkEnvironment getNetworkEnvironment() {
- return networkEnvironment;
- }
-
- public TaskManagerLocation getTaskManagerLocation() {
- return taskManagerLocation;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0a32c6c1/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerRunner.java
new file mode 100644
index 0000000..4f756fb
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerRunner.java
@@ -0,0 +1,749 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskmanager;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.core.memory.HeapMemorySegment;
+import org.apache.flink.core.memory.HybridMemorySegment;
+import org.apache.flink.core.memory.MemorySegmentFactory;
+import org.apache.flink.core.memory.MemoryType;
+import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.io.disk.iomanager.IOManager;
+import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync;
+import org.apache.flink.runtime.io.network.ConnectionManager;
+import org.apache.flink.runtime.io.network.LocalConnectionManager;
+import org.apache.flink.runtime.io.network.NetworkEnvironment;
+import org.apache.flink.runtime.io.network.TaskEventDispatcher;
+import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
+import org.apache.flink.runtime.io.network.netty.NettyConfig;
+import org.apache.flink.runtime.io.network.netty.NettyConnectionManager;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionManager;
+import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.runtime.memory.MemoryManager;
+import org.apache.flink.runtime.query.KvStateRegistry;
+import org.apache.flink.runtime.query.netty.DisabledKvStateRequestStats;
+import org.apache.flink.runtime.query.netty.KvStateServer;
+import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
+import org.apache.flink.runtime.taskexecutor.TaskExecutor;
+import org.apache.flink.runtime.taskexecutor.TaskExecutorConfiguration;
+import org.apache.flink.runtime.util.EnvironmentInformation;
+import org.apache.flink.runtime.util.LeaderRetrievalUtils;
+import org.apache.flink.util.MathUtils;
+import org.apache.flink.util.NetUtils;
+
+import akka.actor.ActorSystem;
+import akka.util.Timeout;
+import com.typesafe.config.Config;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.Option;
+import scala.Some;
+import scala.Tuple2;
+import scala.concurrent.duration.Duration;
+import scala.concurrent.duration.FiniteDuration;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * This class is the executable entry point for the task manager in yarn or standalone mode.
+ * It constructs the related components (network, I/O manager, memory manager, RPC service, HA service)
+ * and starts them.
+ */
+public class TaskManagerRunner {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TaskManagerRunner.class);
+
+ /**
+ * Constructs related components of the TaskManager and starts them.
+ *
+ * @param configuration The configuration for the TaskManager.
+ * @param resourceID The id of the resource which the task manager will run on.
+ * @param rpcService Optionally, The rpc service which is used to start and connect to the TaskManager RpcEndpoint .
+ * If none is given, then a RpcService is constructed from the configuration.
+ * @param taskManagerHostname Optionally, The hostname/address that describes the TaskManager's data location.
+ * If none is given, it can be got from the configuration.
+ * @param localTaskManagerCommunication If true, the TaskManager will not initiate the TCP network stack.
+ * @param haServices Optionally, a high availability service can be provided. If none is given,
+ * then a HighAvailabilityServices is constructed from the configuration.
+ */
+ public static void createAndStartComponents(
+ final Configuration configuration,
+ final ResourceID resourceID,
+ RpcService rpcService,
+ String taskManagerHostname,
+ boolean localTaskManagerCommunication,
+ HighAvailabilityServices haServices) throws Exception {
+
+ checkNotNull(configuration);
+ checkNotNull(resourceID);
+
+ if (taskManagerHostname == null || taskManagerHostname.isEmpty()) {
+ taskManagerHostname = selectNetworkInterface(configuration);
+ }
+
+ if (rpcService == null) {
+ // if no task manager port has been configured, use 0 (system will pick any free port)
+ final int actorSystemPort = configuration.getInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, 0);
+ if (actorSystemPort < 0 || actorSystemPort > 65535) {
+ throw new IllegalConfigurationException("Invalid value for '" +
+ ConfigConstants.TASK_MANAGER_IPC_PORT_KEY +
+ "' (port for the TaskManager actor system) : " + actorSystemPort +
+ " - Leave config parameter empty or use 0 to let the system choose a port automatically.");
+ }
+ rpcService = createRpcService(configuration, taskManagerHostname, actorSystemPort);
+ }
+
+ if(haServices == null) {
+ // start high availability service to implement getResourceManagerLeaderRetriever method only
+ haServices = new HighAvailabilityServices() {
+ @Override
+ public LeaderRetrievalService getResourceManagerLeaderRetriever() throws Exception {
+ return LeaderRetrievalUtils.createLeaderRetrievalService(configuration);
+ }
+
+ @Override
+ public LeaderElectionService getResourceManagerLeaderElectionService() throws Exception {
+ return null;
+ }
+
+ @Override
+ public LeaderElectionService getJobMasterLeaderElectionService(JobID jobID) throws Exception {
+ return null;
+ }
+ };
+ }
+
+ createAndStartTaskManagerComponents(
+ configuration,
+ resourceID,
+ rpcService,
+ taskManagerHostname,
+ haServices,
+ localTaskManagerCommunication);
+ }
+
+ /**
+ * <p/>
+ * This method tries to select the network interface to use for the TaskManager
+ * communication. The network interface is used both for the actor communication
+ * (coordination) as well as for the data exchange between task managers. Unless
+ * the hostname/interface is explicitly configured in the configuration, this
+ * method will try out various interfaces and methods to connect to the JobManager
+ * and select the one where the connection attempt is successful.
+ * <p/>
+ *
+ * @param configuration The configuration for the TaskManager.
+ * @return The host name under which the TaskManager communicates.
+ */
+ private static String selectNetworkInterface(Configuration configuration) throws Exception {
+ String taskManagerHostname = configuration.getString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, null);
+ if (taskManagerHostname != null) {
+ LOG.info("Using configured hostname/address for TaskManager: " + taskManagerHostname);
+ } else {
+ LeaderRetrievalService leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(configuration);
+ FiniteDuration lookupTimeout = AkkaUtils.getLookupTimeout(configuration);
+
+ InetAddress taskManagerAddress = LeaderRetrievalUtils.findConnectingAddress(leaderRetrievalService, lookupTimeout);
+ taskManagerHostname = taskManagerAddress.getHostName();
+ LOG.info("TaskManager will use hostname/address '{}' ({}) for communication.",
+ taskManagerHostname, taskManagerAddress.getHostAddress());
+ }
+
+ return taskManagerHostname;
+ }
+
+ /**
+ * Utility method to create RPC service from configuration and hostname, port.
+ *
+ * @param configuration The configuration for the TaskManager.
+ * @param taskManagerHostname The hostname/address that describes the TaskManager's data location.
+ * @param actorSystemPort If true, the TaskManager will not initiate the TCP network stack.
+ * @return The rpc service which is used to start and connect to the TaskManager RpcEndpoint .
+ * @throws java.io.IOException Thrown, if the actor system can not bind to the address
+ * @throws java.lang.Exception Thrown is some other error occurs while creating akka actor system
+ */
+ private static RpcService createRpcService(Configuration configuration, String taskManagerHostname, int actorSystemPort)
+ throws Exception{
+
+ // Bring up the TaskManager actor system first, bind it to the given address.
+
+ LOG.info("Starting TaskManager actor system at " +
+ NetUtils.hostAndPortToUrlString(taskManagerHostname, actorSystemPort));
+
+ final ActorSystem taskManagerSystem;
+ try {
+ Tuple2<String, Object> address = new Tuple2<String, Object>(taskManagerHostname, actorSystemPort);
+ Config akkaConfig = AkkaUtils.getAkkaConfig(configuration, new Some<>(address));
+ LOG.debug("Using akka configuration\n " + akkaConfig);
+ taskManagerSystem = AkkaUtils.createActorSystem(akkaConfig);
+ } catch (Throwable t) {
+ if (t instanceof org.jboss.netty.channel.ChannelException) {
+ Throwable cause = t.getCause();
+ if (cause != null && t.getCause() instanceof java.net.BindException) {
+ String address = NetUtils.hostAndPortToUrlString(taskManagerHostname, actorSystemPort);
+ throw new IOException("Unable to bind TaskManager actor system to address " +
+ address + " - " + cause.getMessage(), t);
+ }
+ }
+ throw new Exception("Could not create TaskManager actor system", t);
+ }
+
+ // start akka rpc service based on actor system
+ final Timeout timeout = new Timeout(AkkaUtils.getTimeout(configuration).toMillis(), TimeUnit.MILLISECONDS);
+ final AkkaRpcService akkaRpcService = new AkkaRpcService(taskManagerSystem, timeout);
+
+ return akkaRpcService;
+ }
+
+ /**
+ * @param configuration The configuration for the TaskManager.
+ * @param resourceID The id of the resource which the task manager will run on.
+ * @param rpcService The rpc service which is used to start and connect to the TaskManager RpcEndpoint .
+ * @param taskManagerHostname The hostname/address that describes the TaskManager's data location.
+ * @param haServices Optionally, a high availability service can be provided. If none is given,
+ * then a HighAvailabilityServices is constructed from the configuration.
+ * @param localTaskManagerCommunication If true, the TaskManager will not initiate the TCP network stack.
+ * @throws IllegalConfigurationException Thrown, if the given config contains illegal values.
+ * @throws IOException Thrown, if any of the I/O components (such as buffer pools, I/O manager, ...)
+ * cannot be properly started.
+ * @throws Exception Thrown is some other error occurs while parsing the configuration or
+ * starting the TaskManager components.
+ */
+ private static void createAndStartTaskManagerComponents(
+ Configuration configuration,
+ ResourceID resourceID,
+ RpcService rpcService,
+ String taskManagerHostname,
+ HighAvailabilityServices haServices,
+ boolean localTaskManagerCommunication) throws Exception {
+
+ final TaskExecutorConfiguration taskManagerConfig = parseTaskManagerConfiguration(
+ configuration, taskManagerHostname, localTaskManagerCommunication);
+
+ TaskManagerComponents taskManagerComponents = createTaskManagerComponents(
+ resourceID,
+ InetAddress.getByName(taskManagerHostname),
+ taskManagerConfig,
+ configuration);
+
+ final TaskExecutor taskExecutor = new TaskExecutor(
+ taskManagerConfig,
+ taskManagerComponents.getTaskManagerLocation(),
+ rpcService, taskManagerComponents.getMemoryManager(),
+ taskManagerComponents.getIOManager(),
+ taskManagerComponents.getNetworkEnvironment(),
+ haServices);
+
+ taskExecutor.start();
+ }
+
+ /**
+ * Creates and returns the task manager components.
+ *
+ * @param resourceID resource ID of the task manager
+ * @param taskManagerAddress address of the task manager
+ * @param taskExecutorConfig task manager configuration
+ * @param configuration of Flink
+ * @return task manager components
+ * @throws Exception
+ */
+ private static TaskManagerComponents createTaskManagerComponents(
+ ResourceID resourceID,
+ InetAddress taskManagerAddress,
+ TaskExecutorConfiguration taskExecutorConfig,
+ Configuration configuration) throws Exception {
+
+ MemoryType memType = taskExecutorConfig.getNetworkConfig().memoryType();
+
+ // pre-start checks
+ checkTempDirs(taskExecutorConfig.getTmpDirPaths());
+
+ NetworkEnvironmentConfiguration networkEnvironmentConfiguration = taskExecutorConfig.getNetworkConfig();
+
+ NetworkBufferPool networkBufferPool = new NetworkBufferPool(
+ networkEnvironmentConfiguration.numNetworkBuffers(),
+ networkEnvironmentConfiguration.networkBufferSize(),
+ networkEnvironmentConfiguration.memoryType());
+
+ ConnectionManager connectionManager;
+
+ if (networkEnvironmentConfiguration.nettyConfig().isDefined()) {
+ connectionManager = new NettyConnectionManager(networkEnvironmentConfiguration.nettyConfig().get());
+ } else {
+ connectionManager = new LocalConnectionManager();
+ }
+
+ ResultPartitionManager resultPartitionManager = new ResultPartitionManager();
+ TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
+
+ KvStateRegistry kvStateRegistry = new KvStateRegistry();
+
+ KvStateServer kvStateServer;
+
+ if (networkEnvironmentConfiguration.nettyConfig().isDefined()) {
+ NettyConfig nettyConfig = networkEnvironmentConfiguration.nettyConfig().get();
+
+ int numNetworkThreads = networkEnvironmentConfiguration.queryServerNetworkThreads() == 0 ?
+ nettyConfig.getNumberOfSlots() : networkEnvironmentConfiguration.queryServerNetworkThreads();
+
+ int numQueryThreads = networkEnvironmentConfiguration.queryServerQueryThreads() == 0 ?
+ nettyConfig.getNumberOfSlots() : networkEnvironmentConfiguration.queryServerQueryThreads();
+
+ kvStateServer = new KvStateServer(
+ taskManagerAddress,
+ networkEnvironmentConfiguration.queryServerPort(),
+ numNetworkThreads,
+ numQueryThreads,
+ kvStateRegistry,
+ new DisabledKvStateRequestStats());
+ } else {
+ kvStateServer = null;
+ }
+
+ // we start the network first, to make sure it can allocate its buffers first
+ final NetworkEnvironment network = new NetworkEnvironment(
+ networkBufferPool,
+ connectionManager,
+ resultPartitionManager,
+ taskEventDispatcher,
+ kvStateRegistry,
+ kvStateServer,
+ networkEnvironmentConfiguration.ioMode(),
+ networkEnvironmentConfiguration.partitionRequestInitialBackoff(),
+ networkEnvironmentConfiguration.partitinRequestMaxBackoff());
+
+ network.start();
+
+ final TaskManagerLocation taskManagerLocation = new TaskManagerLocation(
+ resourceID,
+ taskManagerAddress,
+ network.getConnectionManager().getDataPort());
+
+ // computing the amount of memory to use depends on how much memory is available
+ // it strictly needs to happen AFTER the network stack has been initialized
+
+ // check if a value has been configured
+ long configuredMemory = configuration.getLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
+ checkConfigParameter(configuredMemory == -1 || configuredMemory > 0, configuredMemory,
+ ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY,
+ "MemoryManager needs at least one MB of memory. " +
+ "If you leave this config parameter empty, the system automatically " +
+ "pick a fraction of the available memory.");
+
+ final long memorySize;
+ boolean preAllocateMemory = configuration.getBoolean(
+ ConfigConstants.TASK_MANAGER_MEMORY_PRE_ALLOCATE_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_PRE_ALLOCATE);
+ if (configuredMemory > 0) {
+ if (preAllocateMemory) {
+ LOG.info("Using {} MB for managed memory." , configuredMemory);
+ } else {
+ LOG.info("Limiting managed memory to {} MB, memory will be allocated lazily." , configuredMemory);
+ }
+ memorySize = configuredMemory << 20; // megabytes to bytes
+ } else {
+ float fraction = configuration.getFloat(
+ ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY,
+ ConfigConstants.DEFAULT_MEMORY_MANAGER_MEMORY_FRACTION);
+ checkConfigParameter(fraction > 0.0f && fraction < 1.0f, fraction,
+ ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY,
+ "MemoryManager fraction of the free memory must be between 0.0 and 1.0");
+
+ if (memType == MemoryType.HEAP) {
+ long relativeMemSize = (long) (EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag() * fraction);
+ if (preAllocateMemory) {
+ LOG.info("Using {} of the currently free heap space for managed heap memory ({} MB)." ,
+ fraction , relativeMemSize >> 20);
+ } else {
+ LOG.info("Limiting managed memory to {} of the currently free heap space ({} MB), " +
+ "memory will be allocated lazily." , fraction , relativeMemSize >> 20);
+ }
+ memorySize = relativeMemSize;
+ } else if (memType == MemoryType.OFF_HEAP) {
+ // The maximum heap memory has been adjusted according to the fraction
+ long maxMemory = EnvironmentInformation.getMaxJvmHeapMemory();
+ long directMemorySize = (long) (maxMemory / (1.0 - fraction) * fraction);
+ if (preAllocateMemory) {
+ LOG.info("Using {} of the maximum memory size for managed off-heap memory ({} MB)." ,
+ fraction, directMemorySize >> 20);
+ } else {
+ LOG.info("Limiting managed memory to {} of the maximum memory size ({} MB)," +
+ " memory will be allocated lazily.", fraction, directMemorySize >> 20);
+ }
+ memorySize = directMemorySize;
+ } else {
+ throw new RuntimeException("No supported memory type detected.");
+ }
+ }
+
+ // now start the memory manager
+ final MemoryManager memoryManager;
+ try {
+ memoryManager = new MemoryManager(
+ memorySize,
+ taskExecutorConfig.getNumberOfSlots(),
+ taskExecutorConfig.getNetworkConfig().networkBufferSize(),
+ memType,
+ preAllocateMemory);
+ } catch (OutOfMemoryError e) {
+ if (memType == MemoryType.HEAP) {
+ throw new Exception("OutOfMemory error (" + e.getMessage() +
+ ") while allocating the TaskManager heap memory (" + memorySize + " bytes).", e);
+ } else if (memType == MemoryType.OFF_HEAP) {
+ throw new Exception("OutOfMemory error (" + e.getMessage() +
+ ") while allocating the TaskManager off-heap memory (" + memorySize +
+ " bytes).Try increasing the maximum direct memory (-XX:MaxDirectMemorySize)", e);
+ } else {
+ throw e;
+ }
+ }
+
+ // start the I/O manager, it will create some temp directories.
+ final IOManager ioManager = new IOManagerAsync(taskExecutorConfig.getTmpDirPaths());
+
+ return new TaskManagerComponents(taskManagerLocation, memoryManager, ioManager, network);
+ }
+
+ // --------------------------------------------------------------------------
+ // Parsing and checking the TaskManager Configuration
+ // --------------------------------------------------------------------------
+
+ /**
+ * Utility method to extract TaskManager config parameters from the configuration and to
+ * sanity check them.
+ *
+ * @param configuration The configuration.
+ * @param taskManagerHostname The host name under which the TaskManager communicates.
+ * @param localTaskManagerCommunication True, to skip initializing the network stack.
+ * Use only in cases where only one task manager runs.
+ * @return TaskExecutorConfiguration that wrappers InstanceConnectionInfo, NetworkEnvironmentConfiguration, etc.
+ */
+ private static TaskExecutorConfiguration parseTaskManagerConfiguration(
+ Configuration configuration,
+ String taskManagerHostname,
+ boolean localTaskManagerCommunication) throws Exception {
+
+ // ------- read values from the config and check them ---------
+ // (a lot of them)
+
+ // ----> hosts / ports for communication and data exchange
+
+ int dataport = configuration.getInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_DATA_PORT);
+
+ checkConfigParameter(dataport > 0, dataport, ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
+ "Leave config parameter empty or use 0 to let the system choose a port automatically.");
+
+ InetAddress taskManagerAddress = InetAddress.getByName(taskManagerHostname);
+ final InetSocketAddress taskManagerInetSocketAddress = new InetSocketAddress(taskManagerAddress, dataport);
+
+ // ----> memory / network stack (shuffles/broadcasts), task slots, temp directories
+
+ // we need this because many configs have been written with a "-1" entry
+ int slots = configuration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
+ if (slots == -1) {
+ slots = 1;
+ }
+
+ checkConfigParameter(slots >= 1, slots, ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS,
+ "Number of task slots must be at least one.");
+
+ final int numNetworkBuffers = configuration.getInteger(
+ ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS);
+
+ checkConfigParameter(numNetworkBuffers > 0, numNetworkBuffers,
+ ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, "");
+
+ final int pageSize = configuration.getInteger(
+ ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_SEGMENT_SIZE);
+
+ // check page size of for minimum size
+ checkConfigParameter(pageSize >= MemoryManager.MIN_PAGE_SIZE, pageSize,
+ ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
+ "Minimum memory segment size is " + MemoryManager.MIN_PAGE_SIZE);
+
+ // check page size for power of two
+ checkConfigParameter(MathUtils.isPowerOf2(pageSize), pageSize,
+ ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
+ "Memory segment size must be a power of 2.");
+
+ // check whether we use heap or off-heap memory
+ final MemoryType memType;
+ if (configuration.getBoolean(ConfigConstants.TASK_MANAGER_MEMORY_OFF_HEAP_KEY, false)) {
+ memType = MemoryType.OFF_HEAP;
+ } else {
+ memType = MemoryType.HEAP;
+ }
+
+ // initialize the memory segment factory accordingly
+ if (memType == MemoryType.HEAP) {
+ if (!MemorySegmentFactory.initializeIfNotInitialized(HeapMemorySegment.FACTORY)) {
+ throw new Exception("Memory type is set to heap memory, but memory segment " +
+ "factory has been initialized for off-heap memory segments");
+ }
+ } else {
+ if (!MemorySegmentFactory.initializeIfNotInitialized(HybridMemorySegment.FACTORY)) {
+ throw new Exception("Memory type is set to off-heap memory, but memory segment " +
+ "factory has been initialized for heap memory segments");
+ }
+ }
+
+ final String[] tmpDirs = configuration.getString(
+ ConfigConstants.TASK_MANAGER_TMP_DIR_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH).split(",|" + File.pathSeparator);
+
+ final NettyConfig nettyConfig;
+ if (!localTaskManagerCommunication) {
+ nettyConfig = new NettyConfig(taskManagerInetSocketAddress.getAddress(),
+ taskManagerInetSocketAddress.getPort(), pageSize, slots, configuration);
+ } else {
+ nettyConfig = null;
+ }
+
+ // Default spill I/O mode for intermediate results
+ final String syncOrAsync = configuration.getString(
+ ConfigConstants.TASK_MANAGER_NETWORK_DEFAULT_IO_MODE,
+ ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_DEFAULT_IO_MODE);
+
+ final IOManager.IOMode ioMode;
+ if (syncOrAsync.equals("async")) {
+ ioMode = IOManager.IOMode.ASYNC;
+ } else {
+ ioMode = IOManager.IOMode.SYNC;
+ }
+
+ final int queryServerPort = configuration.getInteger(
+ ConfigConstants.QUERYABLE_STATE_SERVER_PORT,
+ ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_PORT);
+
+ final int queryServerNetworkThreads = configuration.getInteger(
+ ConfigConstants.QUERYABLE_STATE_SERVER_NETWORK_THREADS,
+ ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_NETWORK_THREADS);
+
+ final int queryServerQueryThreads = configuration.getInteger(
+ ConfigConstants.QUERYABLE_STATE_SERVER_QUERY_THREADS,
+ ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_QUERY_THREADS);
+
+ final NetworkEnvironmentConfiguration networkConfig = new NetworkEnvironmentConfiguration(
+ numNetworkBuffers,
+ pageSize,
+ memType,
+ ioMode,
+ queryServerPort,
+ queryServerNetworkThreads,
+ queryServerQueryThreads,
+ Option.apply(nettyConfig),
+ 500,
+ 3000);
+
+ // ----> timeouts, library caching, profiling
+
+ final FiniteDuration timeout;
+ try {
+ timeout = AkkaUtils.getTimeout(configuration);
+ } catch (Exception e) {
+ throw new IllegalArgumentException(
+ "Invalid format for '" + ConfigConstants.AKKA_ASK_TIMEOUT +
+ "'.Use formats like '50 s' or '1 min' to specify the timeout.");
+ }
+ LOG.info("Messages between TaskManager and JobManager have a max timeout of " + timeout);
+
+ final long cleanupInterval = configuration.getLong(
+ ConfigConstants.LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL,
+ ConfigConstants.DEFAULT_LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL) * 1000;
+
+ final FiniteDuration finiteRegistrationDuration;
+ try {
+ Duration maxRegistrationDuration = Duration.create(configuration.getString(
+ ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION,
+ ConfigConstants.DEFAULT_TASK_MANAGER_MAX_REGISTRATION_DURATION));
+ if (maxRegistrationDuration.isFinite()) {
+ finiteRegistrationDuration = new FiniteDuration(maxRegistrationDuration.toSeconds(), TimeUnit.SECONDS);
+ } else {
+ finiteRegistrationDuration = null;
+ }
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Invalid format for parameter " +
+ ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION, e);
+ }
+
+ final FiniteDuration initialRegistrationPause;
+ try {
+ Duration pause = Duration.create(configuration.getString(
+ ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE,
+ ConfigConstants.DEFAULT_TASK_MANAGER_INITIAL_REGISTRATION_PAUSE));
+ if (pause.isFinite()) {
+ initialRegistrationPause = new FiniteDuration(pause.toSeconds(), TimeUnit.SECONDS);
+ } else {
+ throw new IllegalArgumentException("The initial registration pause must be finite: " + pause);
+ }
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Invalid format for parameter " +
+ ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
+ }
+
+ final FiniteDuration maxRegistrationPause;
+ try {
+ Duration pause = Duration.create(configuration.getString(
+ ConfigConstants.TASK_MANAGER_MAX_REGISTARTION_PAUSE,
+ ConfigConstants.DEFAULT_TASK_MANAGER_MAX_REGISTRATION_PAUSE));
+ if (pause.isFinite()) {
+ maxRegistrationPause = new FiniteDuration(pause.toSeconds(), TimeUnit.SECONDS);
+ } else {
+ throw new IllegalArgumentException("The maximum registration pause must be finite: " + pause);
+ }
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Invalid format for parameter " +
+ ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
+ }
+
+ final FiniteDuration refusedRegistrationPause;
+ try {
+ Duration pause = Duration.create(configuration.getString(
+ ConfigConstants.TASK_MANAGER_REFUSED_REGISTRATION_PAUSE,
+ ConfigConstants.DEFAULT_TASK_MANAGER_REFUSED_REGISTRATION_PAUSE));
+ if (pause.isFinite()) {
+ refusedRegistrationPause = new FiniteDuration(pause.toSeconds(), TimeUnit.SECONDS);
+ } else {
+ throw new IllegalArgumentException("The refused registration pause must be finite: " + pause);
+ }
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Invalid format for parameter " +
+ ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
+ }
+
+ return new TaskExecutorConfiguration(
+ tmpDirs,
+ cleanupInterval,
+ networkConfig,
+ timeout,
+ finiteRegistrationDuration,
+ slots,
+ configuration,
+ initialRegistrationPause,
+ maxRegistrationPause,
+ refusedRegistrationPause);
+ }
+
+ /**
+ * Validates a condition for a config parameter and displays a standard exception, if the
+ * the condition does not hold.
+ *
+ * @param condition The condition that must hold. If the condition is false, an exception is thrown.
+ * @param parameter The parameter value. Will be shown in the exception message.
+ * @param name The name of the config parameter. Will be shown in the exception message.
+ * @param errorMessage The optional custom error message to append to the exception message.
+ */
+ private static void checkConfigParameter(
+ boolean condition,
+ Object parameter,
+ String name,
+ String errorMessage) {
+ if (!condition) {
+ throw new IllegalConfigurationException("Invalid configuration value for " + name + " : " + parameter + " - " + errorMessage);
+ }
+ }
+
+ /**
+ * Validates that all the directories denoted by the strings do actually exist, are proper
+ * directories (not files), and are writable.
+ *
+ * @param tmpDirs The array of directory paths to check.
+ * @throws Exception Thrown if any of the directories does not exist or is not writable
+ * or is a file, rather than a directory.
+ */
+ private static void checkTempDirs(String[] tmpDirs) throws IOException {
+ for (String dir : tmpDirs) {
+ if (dir != null && !dir.equals("")) {
+ File file = new File(dir);
+ if (!file.exists()) {
+ throw new IOException("Temporary file directory " + file.getAbsolutePath() + " does not exist.");
+ }
+ if (!file.isDirectory()) {
+ throw new IOException("Temporary file directory " + file.getAbsolutePath() + " is not a directory.");
+ }
+ if (!file.canWrite()) {
+ throw new IOException("Temporary file directory " + file.getAbsolutePath() + " is not writable.");
+ }
+
+ if (LOG.isInfoEnabled()) {
+ long totalSpaceGb = file.getTotalSpace() >> 30;
+ long usableSpaceGb = file.getUsableSpace() >> 30;
+ double usablePercentage = (double)usableSpaceGb / totalSpaceGb * 100;
+ String path = file.getAbsolutePath();
+ LOG.info(String.format("Temporary file directory '%s': total %d GB, " + "usable %d GB (%.2f%% usable)",
+ path, totalSpaceGb, usableSpaceGb, usablePercentage));
+ }
+ } else {
+ throw new IllegalArgumentException("Temporary file directory #$id is null.");
+ }
+ }
+ }
+
+ private static class TaskManagerComponents {
+ private final TaskManagerLocation taskManagerLocation;
+ private final MemoryManager memoryManager;
+ private final IOManager ioManager;
+ private final NetworkEnvironment networkEnvironment;
+
+ private TaskManagerComponents(
+ TaskManagerLocation taskManagerLocation,
+ MemoryManager memoryManager,
+ IOManager ioManager,
+ NetworkEnvironment networkEnvironment) {
+
+ this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
+ this.memoryManager = Preconditions.checkNotNull(memoryManager);
+ this.ioManager = Preconditions.checkNotNull(ioManager);
+ this.networkEnvironment = Preconditions.checkNotNull(networkEnvironment);
+ }
+
+ public MemoryManager getMemoryManager() {
+ return memoryManager;
+ }
+
+ public IOManager getIOManager() {
+ return ioManager;
+ }
+
+ public NetworkEnvironment getNetworkEnvironment() {
+ return networkEnvironment;
+ }
+
+ public TaskManagerLocation getTaskManagerLocation() {
+ return taskManagerLocation;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0a32c6c1/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index 09aab18..26218dd 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -19,17 +19,22 @@
package org.apache.flink.runtime.taskexecutor;
import org.apache.flink.api.common.time.Time;
-import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.highavailability.NonHaServices;
import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
+import org.apache.flink.runtime.io.disk.iomanager.IOManager;
+import org.apache.flink.runtime.io.network.NetworkEnvironment;
import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
+import org.apache.flink.runtime.memory.MemoryManager;
import org.apache.flink.runtime.rpc.TestingRpcService;
import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
+import org.apache.flink.runtime.rpc.TestingSerialRpcService;
+import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.util.TestLogger;
import org.junit.Test;
+import org.powermock.api.mockito.PowerMockito;
import java.util.UUID;
import static org.junit.Assert.*;
@@ -42,19 +47,31 @@ public class TaskExecutorTest extends TestLogger {
final ResourceID resourceID = ResourceID.generate();
final String resourceManagerAddress = "/resource/manager/address/one";
- final TestingRpcService rpc = new TestingRpcService();
+ final TestingSerialRpcService rpc = new TestingSerialRpcService();
try {
// register a mock resource manager gateway
ResourceManagerGateway rmGateway = mock(ResourceManagerGateway.class);
+ TaskExecutorConfiguration taskExecutorConfiguration = mock(TaskExecutorConfiguration.class);
+ PowerMockito.when(taskExecutorConfiguration.getNumberOfSlots()).thenReturn(1);
rpc.registerGateway(resourceManagerAddress, rmGateway);
+ TaskManagerLocation taskManagerLocation = mock(TaskManagerLocation.class);
+ when(taskManagerLocation.getResourceID()).thenReturn(resourceID);
+
NonHaServices haServices = new NonHaServices(resourceManagerAddress);
- TaskExecutor taskManager = TaskExecutor.startTaskManagerComponentsAndActor(
- new Configuration(), resourceID, rpc, "localhost", haServices, true);
- String taskManagerAddress = taskManager.getAddress();
+
+ TaskExecutor taskManager = new TaskExecutor(
+ taskExecutorConfiguration,
+ taskManagerLocation,
+ rpc, mock(MemoryManager.class),
+ mock(IOManager.class),
+ mock(NetworkEnvironment.class),
+ haServices);
+
taskManager.start();
+ String taskManagerAddress = taskManager.getAddress();
- verify(rmGateway, timeout(5000)).registerTaskExecutor(
+ verify(rmGateway).registerTaskExecutor(
any(UUID.class), eq(taskManagerAddress), eq(resourceID), any(Time.class));
}
finally {
@@ -71,7 +88,7 @@ public class TaskExecutorTest extends TestLogger {
final UUID leaderId1 = UUID.randomUUID();
final UUID leaderId2 = UUID.randomUUID();
- final TestingRpcService rpc = new TestingRpcService();
+ final TestingSerialRpcService rpc = new TestingSerialRpcService();
try {
// register the mock resource manager gateways
ResourceManagerGateway rmGateway1 = mock(ResourceManagerGateway.class);
@@ -84,10 +101,22 @@ public class TaskExecutorTest extends TestLogger {
TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServices();
haServices.setResourceManagerLeaderRetriever(testLeaderService);
- TaskExecutor taskManager = TaskExecutor.startTaskManagerComponentsAndActor(
- new Configuration(), resourceID, rpc, "localhost", haServices, true);
- String taskManagerAddress = taskManager.getAddress();
+ TaskExecutorConfiguration taskExecutorConfiguration = mock(TaskExecutorConfiguration.class);
+ PowerMockito.when(taskExecutorConfiguration.getNumberOfSlots()).thenReturn(1);
+
+ TaskManagerLocation taskManagerLocation = mock(TaskManagerLocation.class);
+ when(taskManagerLocation.getResourceID()).thenReturn(resourceID);
+
+ TaskExecutor taskManager = new TaskExecutor(
+ taskExecutorConfiguration,
+ taskManagerLocation,
+ rpc, mock(MemoryManager.class),
+ mock(IOManager.class),
+ mock(NetworkEnvironment.class),
+ haServices);
+
taskManager.start();
+ String taskManagerAddress = taskManager.getAddress();
// no connection initially, since there is no leader
assertNull(taskManager.getResourceManagerConnection());
@@ -95,7 +124,7 @@ public class TaskExecutorTest extends TestLogger {
// define a leader and see that a registration happens
testLeaderService.notifyListener(address1, leaderId1);
- verify(rmGateway1, timeout(5000)).registerTaskExecutor(
+ verify(rmGateway1).registerTaskExecutor(
eq(leaderId1), eq(taskManagerAddress), eq(resourceID), any(Time.class));
assertNotNull(taskManager.getResourceManagerConnection());
@@ -105,7 +134,7 @@ public class TaskExecutorTest extends TestLogger {
// set a new leader, see that a registration happens
testLeaderService.notifyListener(address2, leaderId2);
- verify(rmGateway2, timeout(5000)).registerTaskExecutor(
+ verify(rmGateway2).registerTaskExecutor(
eq(leaderId2), eq(taskManagerAddress), eq(resourceID), any(Time.class));
assertNotNull(taskManager.getResourceManagerConnection());
}
[11/50] [abbrv] flink git commit: [FLINK-4505] [cluster mngt]
Separate TaskManager service configuration from TaskManagerConfiguration;
Implement TaskManagerRunner
Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerRunner.java
deleted file mode 100644
index 4f756fb..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerRunner.java
+++ /dev/null
@@ -1,749 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.taskmanager;
-
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.IllegalConfigurationException;
-import org.apache.flink.core.memory.HeapMemorySegment;
-import org.apache.flink.core.memory.HybridMemorySegment;
-import org.apache.flink.core.memory.MemorySegmentFactory;
-import org.apache.flink.core.memory.MemoryType;
-import org.apache.flink.runtime.akka.AkkaUtils;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.io.disk.iomanager.IOManager;
-import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync;
-import org.apache.flink.runtime.io.network.ConnectionManager;
-import org.apache.flink.runtime.io.network.LocalConnectionManager;
-import org.apache.flink.runtime.io.network.NetworkEnvironment;
-import org.apache.flink.runtime.io.network.TaskEventDispatcher;
-import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
-import org.apache.flink.runtime.io.network.netty.NettyConfig;
-import org.apache.flink.runtime.io.network.netty.NettyConnectionManager;
-import org.apache.flink.runtime.io.network.partition.ResultPartitionManager;
-import org.apache.flink.runtime.leaderelection.LeaderElectionService;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
-import org.apache.flink.runtime.memory.MemoryManager;
-import org.apache.flink.runtime.query.KvStateRegistry;
-import org.apache.flink.runtime.query.netty.DisabledKvStateRequestStats;
-import org.apache.flink.runtime.query.netty.KvStateServer;
-import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
-import org.apache.flink.runtime.taskexecutor.TaskExecutor;
-import org.apache.flink.runtime.taskexecutor.TaskExecutorConfiguration;
-import org.apache.flink.runtime.util.EnvironmentInformation;
-import org.apache.flink.runtime.util.LeaderRetrievalUtils;
-import org.apache.flink.util.MathUtils;
-import org.apache.flink.util.NetUtils;
-
-import akka.actor.ActorSystem;
-import akka.util.Timeout;
-import com.typesafe.config.Config;
-import org.apache.flink.util.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.Option;
-import scala.Some;
-import scala.Tuple2;
-import scala.concurrent.duration.Duration;
-import scala.concurrent.duration.FiniteDuration;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/**
- * This class is the executable entry point for the task manager in yarn or standalone mode.
- * It constructs the related components (network, I/O manager, memory manager, RPC service, HA service)
- * and starts them.
- */
-public class TaskManagerRunner {
-
- private static final Logger LOG = LoggerFactory.getLogger(TaskManagerRunner.class);
-
- /**
- * Constructs related components of the TaskManager and starts them.
- *
- * @param configuration The configuration for the TaskManager.
- * @param resourceID The id of the resource which the task manager will run on.
- * @param rpcService Optionally, The rpc service which is used to start and connect to the TaskManager RpcEndpoint .
- * If none is given, then a RpcService is constructed from the configuration.
- * @param taskManagerHostname Optionally, The hostname/address that describes the TaskManager's data location.
- * If none is given, it can be got from the configuration.
- * @param localTaskManagerCommunication If true, the TaskManager will not initiate the TCP network stack.
- * @param haServices Optionally, a high availability service can be provided. If none is given,
- * then a HighAvailabilityServices is constructed from the configuration.
- */
- public static void createAndStartComponents(
- final Configuration configuration,
- final ResourceID resourceID,
- RpcService rpcService,
- String taskManagerHostname,
- boolean localTaskManagerCommunication,
- HighAvailabilityServices haServices) throws Exception {
-
- checkNotNull(configuration);
- checkNotNull(resourceID);
-
- if (taskManagerHostname == null || taskManagerHostname.isEmpty()) {
- taskManagerHostname = selectNetworkInterface(configuration);
- }
-
- if (rpcService == null) {
- // if no task manager port has been configured, use 0 (system will pick any free port)
- final int actorSystemPort = configuration.getInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, 0);
- if (actorSystemPort < 0 || actorSystemPort > 65535) {
- throw new IllegalConfigurationException("Invalid value for '" +
- ConfigConstants.TASK_MANAGER_IPC_PORT_KEY +
- "' (port for the TaskManager actor system) : " + actorSystemPort +
- " - Leave config parameter empty or use 0 to let the system choose a port automatically.");
- }
- rpcService = createRpcService(configuration, taskManagerHostname, actorSystemPort);
- }
-
- if(haServices == null) {
- // start high availability service to implement getResourceManagerLeaderRetriever method only
- haServices = new HighAvailabilityServices() {
- @Override
- public LeaderRetrievalService getResourceManagerLeaderRetriever() throws Exception {
- return LeaderRetrievalUtils.createLeaderRetrievalService(configuration);
- }
-
- @Override
- public LeaderElectionService getResourceManagerLeaderElectionService() throws Exception {
- return null;
- }
-
- @Override
- public LeaderElectionService getJobMasterLeaderElectionService(JobID jobID) throws Exception {
- return null;
- }
- };
- }
-
- createAndStartTaskManagerComponents(
- configuration,
- resourceID,
- rpcService,
- taskManagerHostname,
- haServices,
- localTaskManagerCommunication);
- }
-
- /**
- * <p/>
- * This method tries to select the network interface to use for the TaskManager
- * communication. The network interface is used both for the actor communication
- * (coordination) as well as for the data exchange between task managers. Unless
- * the hostname/interface is explicitly configured in the configuration, this
- * method will try out various interfaces and methods to connect to the JobManager
- * and select the one where the connection attempt is successful.
- * <p/>
- *
- * @param configuration The configuration for the TaskManager.
- * @return The host name under which the TaskManager communicates.
- */
- private static String selectNetworkInterface(Configuration configuration) throws Exception {
- String taskManagerHostname = configuration.getString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, null);
- if (taskManagerHostname != null) {
- LOG.info("Using configured hostname/address for TaskManager: " + taskManagerHostname);
- } else {
- LeaderRetrievalService leaderRetrievalService = LeaderRetrievalUtils.createLeaderRetrievalService(configuration);
- FiniteDuration lookupTimeout = AkkaUtils.getLookupTimeout(configuration);
-
- InetAddress taskManagerAddress = LeaderRetrievalUtils.findConnectingAddress(leaderRetrievalService, lookupTimeout);
- taskManagerHostname = taskManagerAddress.getHostName();
- LOG.info("TaskManager will use hostname/address '{}' ({}) for communication.",
- taskManagerHostname, taskManagerAddress.getHostAddress());
- }
-
- return taskManagerHostname;
- }
-
- /**
- * Utility method to create RPC service from configuration and hostname, port.
- *
- * @param configuration The configuration for the TaskManager.
- * @param taskManagerHostname The hostname/address that describes the TaskManager's data location.
- * @param actorSystemPort If true, the TaskManager will not initiate the TCP network stack.
- * @return The rpc service which is used to start and connect to the TaskManager RpcEndpoint .
- * @throws java.io.IOException Thrown, if the actor system can not bind to the address
- * @throws java.lang.Exception Thrown is some other error occurs while creating akka actor system
- */
- private static RpcService createRpcService(Configuration configuration, String taskManagerHostname, int actorSystemPort)
- throws Exception{
-
- // Bring up the TaskManager actor system first, bind it to the given address.
-
- LOG.info("Starting TaskManager actor system at " +
- NetUtils.hostAndPortToUrlString(taskManagerHostname, actorSystemPort));
-
- final ActorSystem taskManagerSystem;
- try {
- Tuple2<String, Object> address = new Tuple2<String, Object>(taskManagerHostname, actorSystemPort);
- Config akkaConfig = AkkaUtils.getAkkaConfig(configuration, new Some<>(address));
- LOG.debug("Using akka configuration\n " + akkaConfig);
- taskManagerSystem = AkkaUtils.createActorSystem(akkaConfig);
- } catch (Throwable t) {
- if (t instanceof org.jboss.netty.channel.ChannelException) {
- Throwable cause = t.getCause();
- if (cause != null && t.getCause() instanceof java.net.BindException) {
- String address = NetUtils.hostAndPortToUrlString(taskManagerHostname, actorSystemPort);
- throw new IOException("Unable to bind TaskManager actor system to address " +
- address + " - " + cause.getMessage(), t);
- }
- }
- throw new Exception("Could not create TaskManager actor system", t);
- }
-
- // start akka rpc service based on actor system
- final Timeout timeout = new Timeout(AkkaUtils.getTimeout(configuration).toMillis(), TimeUnit.MILLISECONDS);
- final AkkaRpcService akkaRpcService = new AkkaRpcService(taskManagerSystem, timeout);
-
- return akkaRpcService;
- }
-
- /**
- * @param configuration The configuration for the TaskManager.
- * @param resourceID The id of the resource which the task manager will run on.
- * @param rpcService The rpc service which is used to start and connect to the TaskManager RpcEndpoint .
- * @param taskManagerHostname The hostname/address that describes the TaskManager's data location.
- * @param haServices Optionally, a high availability service can be provided. If none is given,
- * then a HighAvailabilityServices is constructed from the configuration.
- * @param localTaskManagerCommunication If true, the TaskManager will not initiate the TCP network stack.
- * @throws IllegalConfigurationException Thrown, if the given config contains illegal values.
- * @throws IOException Thrown, if any of the I/O components (such as buffer pools, I/O manager, ...)
- * cannot be properly started.
- * @throws Exception Thrown is some other error occurs while parsing the configuration or
- * starting the TaskManager components.
- */
- private static void createAndStartTaskManagerComponents(
- Configuration configuration,
- ResourceID resourceID,
- RpcService rpcService,
- String taskManagerHostname,
- HighAvailabilityServices haServices,
- boolean localTaskManagerCommunication) throws Exception {
-
- final TaskExecutorConfiguration taskManagerConfig = parseTaskManagerConfiguration(
- configuration, taskManagerHostname, localTaskManagerCommunication);
-
- TaskManagerComponents taskManagerComponents = createTaskManagerComponents(
- resourceID,
- InetAddress.getByName(taskManagerHostname),
- taskManagerConfig,
- configuration);
-
- final TaskExecutor taskExecutor = new TaskExecutor(
- taskManagerConfig,
- taskManagerComponents.getTaskManagerLocation(),
- rpcService, taskManagerComponents.getMemoryManager(),
- taskManagerComponents.getIOManager(),
- taskManagerComponents.getNetworkEnvironment(),
- haServices);
-
- taskExecutor.start();
- }
-
- /**
- * Creates and returns the task manager components.
- *
- * @param resourceID resource ID of the task manager
- * @param taskManagerAddress address of the task manager
- * @param taskExecutorConfig task manager configuration
- * @param configuration of Flink
- * @return task manager components
- * @throws Exception
- */
- private static TaskManagerComponents createTaskManagerComponents(
- ResourceID resourceID,
- InetAddress taskManagerAddress,
- TaskExecutorConfiguration taskExecutorConfig,
- Configuration configuration) throws Exception {
-
- MemoryType memType = taskExecutorConfig.getNetworkConfig().memoryType();
-
- // pre-start checks
- checkTempDirs(taskExecutorConfig.getTmpDirPaths());
-
- NetworkEnvironmentConfiguration networkEnvironmentConfiguration = taskExecutorConfig.getNetworkConfig();
-
- NetworkBufferPool networkBufferPool = new NetworkBufferPool(
- networkEnvironmentConfiguration.numNetworkBuffers(),
- networkEnvironmentConfiguration.networkBufferSize(),
- networkEnvironmentConfiguration.memoryType());
-
- ConnectionManager connectionManager;
-
- if (networkEnvironmentConfiguration.nettyConfig().isDefined()) {
- connectionManager = new NettyConnectionManager(networkEnvironmentConfiguration.nettyConfig().get());
- } else {
- connectionManager = new LocalConnectionManager();
- }
-
- ResultPartitionManager resultPartitionManager = new ResultPartitionManager();
- TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
-
- KvStateRegistry kvStateRegistry = new KvStateRegistry();
-
- KvStateServer kvStateServer;
-
- if (networkEnvironmentConfiguration.nettyConfig().isDefined()) {
- NettyConfig nettyConfig = networkEnvironmentConfiguration.nettyConfig().get();
-
- int numNetworkThreads = networkEnvironmentConfiguration.queryServerNetworkThreads() == 0 ?
- nettyConfig.getNumberOfSlots() : networkEnvironmentConfiguration.queryServerNetworkThreads();
-
- int numQueryThreads = networkEnvironmentConfiguration.queryServerQueryThreads() == 0 ?
- nettyConfig.getNumberOfSlots() : networkEnvironmentConfiguration.queryServerQueryThreads();
-
- kvStateServer = new KvStateServer(
- taskManagerAddress,
- networkEnvironmentConfiguration.queryServerPort(),
- numNetworkThreads,
- numQueryThreads,
- kvStateRegistry,
- new DisabledKvStateRequestStats());
- } else {
- kvStateServer = null;
- }
-
- // we start the network first, to make sure it can allocate its buffers first
- final NetworkEnvironment network = new NetworkEnvironment(
- networkBufferPool,
- connectionManager,
- resultPartitionManager,
- taskEventDispatcher,
- kvStateRegistry,
- kvStateServer,
- networkEnvironmentConfiguration.ioMode(),
- networkEnvironmentConfiguration.partitionRequestInitialBackoff(),
- networkEnvironmentConfiguration.partitinRequestMaxBackoff());
-
- network.start();
-
- final TaskManagerLocation taskManagerLocation = new TaskManagerLocation(
- resourceID,
- taskManagerAddress,
- network.getConnectionManager().getDataPort());
-
- // computing the amount of memory to use depends on how much memory is available
- // it strictly needs to happen AFTER the network stack has been initialized
-
- // check if a value has been configured
- long configuredMemory = configuration.getLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
- checkConfigParameter(configuredMemory == -1 || configuredMemory > 0, configuredMemory,
- ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY,
- "MemoryManager needs at least one MB of memory. " +
- "If you leave this config parameter empty, the system automatically " +
- "pick a fraction of the available memory.");
-
- final long memorySize;
- boolean preAllocateMemory = configuration.getBoolean(
- ConfigConstants.TASK_MANAGER_MEMORY_PRE_ALLOCATE_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_PRE_ALLOCATE);
- if (configuredMemory > 0) {
- if (preAllocateMemory) {
- LOG.info("Using {} MB for managed memory." , configuredMemory);
- } else {
- LOG.info("Limiting managed memory to {} MB, memory will be allocated lazily." , configuredMemory);
- }
- memorySize = configuredMemory << 20; // megabytes to bytes
- } else {
- float fraction = configuration.getFloat(
- ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY,
- ConfigConstants.DEFAULT_MEMORY_MANAGER_MEMORY_FRACTION);
- checkConfigParameter(fraction > 0.0f && fraction < 1.0f, fraction,
- ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY,
- "MemoryManager fraction of the free memory must be between 0.0 and 1.0");
-
- if (memType == MemoryType.HEAP) {
- long relativeMemSize = (long) (EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag() * fraction);
- if (preAllocateMemory) {
- LOG.info("Using {} of the currently free heap space for managed heap memory ({} MB)." ,
- fraction , relativeMemSize >> 20);
- } else {
- LOG.info("Limiting managed memory to {} of the currently free heap space ({} MB), " +
- "memory will be allocated lazily." , fraction , relativeMemSize >> 20);
- }
- memorySize = relativeMemSize;
- } else if (memType == MemoryType.OFF_HEAP) {
- // The maximum heap memory has been adjusted according to the fraction
- long maxMemory = EnvironmentInformation.getMaxJvmHeapMemory();
- long directMemorySize = (long) (maxMemory / (1.0 - fraction) * fraction);
- if (preAllocateMemory) {
- LOG.info("Using {} of the maximum memory size for managed off-heap memory ({} MB)." ,
- fraction, directMemorySize >> 20);
- } else {
- LOG.info("Limiting managed memory to {} of the maximum memory size ({} MB)," +
- " memory will be allocated lazily.", fraction, directMemorySize >> 20);
- }
- memorySize = directMemorySize;
- } else {
- throw new RuntimeException("No supported memory type detected.");
- }
- }
-
- // now start the memory manager
- final MemoryManager memoryManager;
- try {
- memoryManager = new MemoryManager(
- memorySize,
- taskExecutorConfig.getNumberOfSlots(),
- taskExecutorConfig.getNetworkConfig().networkBufferSize(),
- memType,
- preAllocateMemory);
- } catch (OutOfMemoryError e) {
- if (memType == MemoryType.HEAP) {
- throw new Exception("OutOfMemory error (" + e.getMessage() +
- ") while allocating the TaskManager heap memory (" + memorySize + " bytes).", e);
- } else if (memType == MemoryType.OFF_HEAP) {
- throw new Exception("OutOfMemory error (" + e.getMessage() +
- ") while allocating the TaskManager off-heap memory (" + memorySize +
- " bytes).Try increasing the maximum direct memory (-XX:MaxDirectMemorySize)", e);
- } else {
- throw e;
- }
- }
-
- // start the I/O manager, it will create some temp directories.
- final IOManager ioManager = new IOManagerAsync(taskExecutorConfig.getTmpDirPaths());
-
- return new TaskManagerComponents(taskManagerLocation, memoryManager, ioManager, network);
- }
-
- // --------------------------------------------------------------------------
- // Parsing and checking the TaskManager Configuration
- // --------------------------------------------------------------------------
-
- /**
- * Utility method to extract TaskManager config parameters from the configuration and to
- * sanity check them.
- *
- * @param configuration The configuration.
- * @param taskManagerHostname The host name under which the TaskManager communicates.
- * @param localTaskManagerCommunication True, to skip initializing the network stack.
- * Use only in cases where only one task manager runs.
- * @return TaskExecutorConfiguration that wrappers InstanceConnectionInfo, NetworkEnvironmentConfiguration, etc.
- */
- private static TaskExecutorConfiguration parseTaskManagerConfiguration(
- Configuration configuration,
- String taskManagerHostname,
- boolean localTaskManagerCommunication) throws Exception {
-
- // ------- read values from the config and check them ---------
- // (a lot of them)
-
- // ----> hosts / ports for communication and data exchange
-
- int dataport = configuration.getInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_DATA_PORT);
-
- checkConfigParameter(dataport > 0, dataport, ConfigConstants.TASK_MANAGER_DATA_PORT_KEY,
- "Leave config parameter empty or use 0 to let the system choose a port automatically.");
-
- InetAddress taskManagerAddress = InetAddress.getByName(taskManagerHostname);
- final InetSocketAddress taskManagerInetSocketAddress = new InetSocketAddress(taskManagerAddress, dataport);
-
- // ----> memory / network stack (shuffles/broadcasts), task slots, temp directories
-
- // we need this because many configs have been written with a "-1" entry
- int slots = configuration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
- if (slots == -1) {
- slots = 1;
- }
-
- checkConfigParameter(slots >= 1, slots, ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS,
- "Number of task slots must be at least one.");
-
- final int numNetworkBuffers = configuration.getInteger(
- ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS);
-
- checkConfigParameter(numNetworkBuffers > 0, numNetworkBuffers,
- ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, "");
-
- final int pageSize = configuration.getInteger(
- ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_SEGMENT_SIZE);
-
- // check page size of for minimum size
- checkConfigParameter(pageSize >= MemoryManager.MIN_PAGE_SIZE, pageSize,
- ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
- "Minimum memory segment size is " + MemoryManager.MIN_PAGE_SIZE);
-
- // check page size for power of two
- checkConfigParameter(MathUtils.isPowerOf2(pageSize), pageSize,
- ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY,
- "Memory segment size must be a power of 2.");
-
- // check whether we use heap or off-heap memory
- final MemoryType memType;
- if (configuration.getBoolean(ConfigConstants.TASK_MANAGER_MEMORY_OFF_HEAP_KEY, false)) {
- memType = MemoryType.OFF_HEAP;
- } else {
- memType = MemoryType.HEAP;
- }
-
- // initialize the memory segment factory accordingly
- if (memType == MemoryType.HEAP) {
- if (!MemorySegmentFactory.initializeIfNotInitialized(HeapMemorySegment.FACTORY)) {
- throw new Exception("Memory type is set to heap memory, but memory segment " +
- "factory has been initialized for off-heap memory segments");
- }
- } else {
- if (!MemorySegmentFactory.initializeIfNotInitialized(HybridMemorySegment.FACTORY)) {
- throw new Exception("Memory type is set to off-heap memory, but memory segment " +
- "factory has been initialized for heap memory segments");
- }
- }
-
- final String[] tmpDirs = configuration.getString(
- ConfigConstants.TASK_MANAGER_TMP_DIR_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH).split(",|" + File.pathSeparator);
-
- final NettyConfig nettyConfig;
- if (!localTaskManagerCommunication) {
- nettyConfig = new NettyConfig(taskManagerInetSocketAddress.getAddress(),
- taskManagerInetSocketAddress.getPort(), pageSize, slots, configuration);
- } else {
- nettyConfig = null;
- }
-
- // Default spill I/O mode for intermediate results
- final String syncOrAsync = configuration.getString(
- ConfigConstants.TASK_MANAGER_NETWORK_DEFAULT_IO_MODE,
- ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_DEFAULT_IO_MODE);
-
- final IOManager.IOMode ioMode;
- if (syncOrAsync.equals("async")) {
- ioMode = IOManager.IOMode.ASYNC;
- } else {
- ioMode = IOManager.IOMode.SYNC;
- }
-
- final int queryServerPort = configuration.getInteger(
- ConfigConstants.QUERYABLE_STATE_SERVER_PORT,
- ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_PORT);
-
- final int queryServerNetworkThreads = configuration.getInteger(
- ConfigConstants.QUERYABLE_STATE_SERVER_NETWORK_THREADS,
- ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_NETWORK_THREADS);
-
- final int queryServerQueryThreads = configuration.getInteger(
- ConfigConstants.QUERYABLE_STATE_SERVER_QUERY_THREADS,
- ConfigConstants.DEFAULT_QUERYABLE_STATE_SERVER_QUERY_THREADS);
-
- final NetworkEnvironmentConfiguration networkConfig = new NetworkEnvironmentConfiguration(
- numNetworkBuffers,
- pageSize,
- memType,
- ioMode,
- queryServerPort,
- queryServerNetworkThreads,
- queryServerQueryThreads,
- Option.apply(nettyConfig),
- 500,
- 3000);
-
- // ----> timeouts, library caching, profiling
-
- final FiniteDuration timeout;
- try {
- timeout = AkkaUtils.getTimeout(configuration);
- } catch (Exception e) {
- throw new IllegalArgumentException(
- "Invalid format for '" + ConfigConstants.AKKA_ASK_TIMEOUT +
- "'.Use formats like '50 s' or '1 min' to specify the timeout.");
- }
- LOG.info("Messages between TaskManager and JobManager have a max timeout of " + timeout);
-
- final long cleanupInterval = configuration.getLong(
- ConfigConstants.LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL,
- ConfigConstants.DEFAULT_LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL) * 1000;
-
- final FiniteDuration finiteRegistrationDuration;
- try {
- Duration maxRegistrationDuration = Duration.create(configuration.getString(
- ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION,
- ConfigConstants.DEFAULT_TASK_MANAGER_MAX_REGISTRATION_DURATION));
- if (maxRegistrationDuration.isFinite()) {
- finiteRegistrationDuration = new FiniteDuration(maxRegistrationDuration.toSeconds(), TimeUnit.SECONDS);
- } else {
- finiteRegistrationDuration = null;
- }
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("Invalid format for parameter " +
- ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION, e);
- }
-
- final FiniteDuration initialRegistrationPause;
- try {
- Duration pause = Duration.create(configuration.getString(
- ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE,
- ConfigConstants.DEFAULT_TASK_MANAGER_INITIAL_REGISTRATION_PAUSE));
- if (pause.isFinite()) {
- initialRegistrationPause = new FiniteDuration(pause.toSeconds(), TimeUnit.SECONDS);
- } else {
- throw new IllegalArgumentException("The initial registration pause must be finite: " + pause);
- }
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("Invalid format for parameter " +
- ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
- }
-
- final FiniteDuration maxRegistrationPause;
- try {
- Duration pause = Duration.create(configuration.getString(
- ConfigConstants.TASK_MANAGER_MAX_REGISTARTION_PAUSE,
- ConfigConstants.DEFAULT_TASK_MANAGER_MAX_REGISTRATION_PAUSE));
- if (pause.isFinite()) {
- maxRegistrationPause = new FiniteDuration(pause.toSeconds(), TimeUnit.SECONDS);
- } else {
- throw new IllegalArgumentException("The maximum registration pause must be finite: " + pause);
- }
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("Invalid format for parameter " +
- ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
- }
-
- final FiniteDuration refusedRegistrationPause;
- try {
- Duration pause = Duration.create(configuration.getString(
- ConfigConstants.TASK_MANAGER_REFUSED_REGISTRATION_PAUSE,
- ConfigConstants.DEFAULT_TASK_MANAGER_REFUSED_REGISTRATION_PAUSE));
- if (pause.isFinite()) {
- refusedRegistrationPause = new FiniteDuration(pause.toSeconds(), TimeUnit.SECONDS);
- } else {
- throw new IllegalArgumentException("The refused registration pause must be finite: " + pause);
- }
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("Invalid format for parameter " +
- ConfigConstants.TASK_MANAGER_INITIAL_REGISTRATION_PAUSE, e);
- }
-
- return new TaskExecutorConfiguration(
- tmpDirs,
- cleanupInterval,
- networkConfig,
- timeout,
- finiteRegistrationDuration,
- slots,
- configuration,
- initialRegistrationPause,
- maxRegistrationPause,
- refusedRegistrationPause);
- }
-
- /**
- * Validates a condition for a config parameter and displays a standard exception, if the
- * the condition does not hold.
- *
- * @param condition The condition that must hold. If the condition is false, an exception is thrown.
- * @param parameter The parameter value. Will be shown in the exception message.
- * @param name The name of the config parameter. Will be shown in the exception message.
- * @param errorMessage The optional custom error message to append to the exception message.
- */
- private static void checkConfigParameter(
- boolean condition,
- Object parameter,
- String name,
- String errorMessage) {
- if (!condition) {
- throw new IllegalConfigurationException("Invalid configuration value for " + name + " : " + parameter + " - " + errorMessage);
- }
- }
-
- /**
- * Validates that all the directories denoted by the strings do actually exist, are proper
- * directories (not files), and are writable.
- *
- * @param tmpDirs The array of directory paths to check.
- * @throws Exception Thrown if any of the directories does not exist or is not writable
- * or is a file, rather than a directory.
- */
- private static void checkTempDirs(String[] tmpDirs) throws IOException {
- for (String dir : tmpDirs) {
- if (dir != null && !dir.equals("")) {
- File file = new File(dir);
- if (!file.exists()) {
- throw new IOException("Temporary file directory " + file.getAbsolutePath() + " does not exist.");
- }
- if (!file.isDirectory()) {
- throw new IOException("Temporary file directory " + file.getAbsolutePath() + " is not a directory.");
- }
- if (!file.canWrite()) {
- throw new IOException("Temporary file directory " + file.getAbsolutePath() + " is not writable.");
- }
-
- if (LOG.isInfoEnabled()) {
- long totalSpaceGb = file.getTotalSpace() >> 30;
- long usableSpaceGb = file.getUsableSpace() >> 30;
- double usablePercentage = (double)usableSpaceGb / totalSpaceGb * 100;
- String path = file.getAbsolutePath();
- LOG.info(String.format("Temporary file directory '%s': total %d GB, " + "usable %d GB (%.2f%% usable)",
- path, totalSpaceGb, usableSpaceGb, usablePercentage));
- }
- } else {
- throw new IllegalArgumentException("Temporary file directory #$id is null.");
- }
- }
- }
-
- private static class TaskManagerComponents {
- private final TaskManagerLocation taskManagerLocation;
- private final MemoryManager memoryManager;
- private final IOManager ioManager;
- private final NetworkEnvironment networkEnvironment;
-
- private TaskManagerComponents(
- TaskManagerLocation taskManagerLocation,
- MemoryManager memoryManager,
- IOManager ioManager,
- NetworkEnvironment networkEnvironment) {
-
- this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
- this.memoryManager = Preconditions.checkNotNull(memoryManager);
- this.ioManager = Preconditions.checkNotNull(ioManager);
- this.networkEnvironment = Preconditions.checkNotNull(networkEnvironment);
- }
-
- public MemoryManager getMemoryManager() {
- return memoryManager;
- }
-
- public IOManager getIOManager() {
- return ioManager;
- }
-
- public NetworkEnvironment getNetworkEnvironment() {
- return networkEnvironment;
- }
-
- public TaskManagerLocation getTaskManagerLocation() {
- return taskManagerLocation;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java
index b6d9306..42655a2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java
@@ -22,6 +22,7 @@ import akka.actor.ActorRef;
import akka.actor.ActorSystem;
import akka.dispatch.Mapper;
import akka.dispatch.OnComplete;
+import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.IllegalConfigurationException;
@@ -170,6 +171,12 @@ public class LeaderRetrievalUtils {
}
public static InetAddress findConnectingAddress(
+ LeaderRetrievalService leaderRetrievalService,
+ Time timeout) throws LeaderRetrievalException {
+ return findConnectingAddress(leaderRetrievalService, new FiniteDuration(timeout.getSize(), timeout.getUnit()));
+ }
+
+ public static InetAddress findConnectingAddress(
LeaderRetrievalService leaderRetrievalService,
FiniteDuration timeout) throws LeaderRetrievalException {
ConnectionUtils.LeaderConnectingAddressListener listener = new ConnectionUtils.LeaderConnectingAddressListener();
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
index 27100fd..a0ca3a1 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
@@ -97,6 +97,10 @@ object AkkaUtils {
createActorSystem(getDefaultAkkaConfig)
}
+ def getAkkaConfig(configuration: Configuration, hostname: String, port: Int): Config = {
+ getAkkaConfig(configuration, if (hostname == null) Some((hostname, port)) else None)
+ }
+
/**
* Creates an akka config with the provided configuration values. If the listening address is
* specified, then the actor system will listen on the respective address.
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.scala
index 893eaa8..97aae34 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/NetworkEnvironmentConfiguration.scala
@@ -30,6 +30,6 @@ case class NetworkEnvironmentConfiguration(
queryServerPort: Int,
queryServerNetworkThreads: Int,
queryServerQueryThreads: Int,
- nettyConfig: Option[NettyConfig] = None,
+ nettyConfig: NettyConfig = null,
partitionRequestInitialBackoff: Int = 500,
partitinRequestMaxBackoff: Int = 3000)
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 5073bb0..c789156 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -1913,7 +1913,7 @@ object TaskManager {
netConfig.networkBufferSize,
netConfig.memoryType)
- val connectionManager = netConfig.nettyConfig match {
+ val connectionManager = Option(netConfig.nettyConfig) match {
case Some(nettyConfig) => new NettyConnectionManager(nettyConfig)
case None => new LocalConnectionManager()
}
@@ -1923,7 +1923,7 @@ object TaskManager {
val kvStateRegistry = new KvStateRegistry()
- val kvStateServer = netConfig.nettyConfig match {
+ val kvStateServer = Option(netConfig.nettyConfig) match {
case Some(nettyConfig) =>
val numNetworkThreads = if (netConfig.queryServerNetworkThreads == 0) {
@@ -2255,7 +2255,7 @@ object TaskManager {
queryServerPort,
queryServerNetworkThreads,
queryServerQueryThreads,
- nettyConfig)
+ nettyConfig.getOrElse(null))
// ----> timeouts, library caching, profiling
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/NetworkEnvironmentTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/NetworkEnvironmentTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/NetworkEnvironmentTest.java
index a9ad75d..cc50b66 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/NetworkEnvironmentTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/NetworkEnvironmentTest.java
@@ -24,7 +24,6 @@ import org.apache.flink.runtime.instance.ActorGateway;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
-import org.apache.flink.runtime.io.network.netty.NettyConfig;
import org.apache.flink.runtime.io.network.partition.ResultPartition;
import org.apache.flink.runtime.io.network.partition.ResultPartitionConsumableNotifier;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
@@ -38,7 +37,6 @@ import org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration;
import org.apache.flink.runtime.taskmanager.Task;
import org.apache.flink.runtime.testingUtils.TestingUtils;
import org.junit.Test;
-import scala.Some;
import scala.concurrent.duration.FiniteDuration;
import scala.concurrent.impl.Promise;
@@ -75,7 +73,7 @@ public class NetworkEnvironmentTest {
0,
0,
0,
- Some.<NettyConfig>empty(),
+ null,
0,
0);
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java
index 5b8e6e6..2a004c5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java
@@ -30,7 +30,6 @@ import java.lang.annotation.Annotation;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
-import java.net.InetAddress;
import java.util.BitSet;
import java.util.UUID;
import java.util.concurrent.Callable;
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index 26218dd..9c1f288 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -26,8 +26,9 @@ import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.network.NetworkEnvironment;
import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
import org.apache.flink.runtime.memory.MemoryManager;
-import org.apache.flink.runtime.rpc.TestingRpcService;
+import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.TestingSerialRpcService;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.util.TestLogger;
@@ -51,8 +52,8 @@ public class TaskExecutorTest extends TestLogger {
try {
// register a mock resource manager gateway
ResourceManagerGateway rmGateway = mock(ResourceManagerGateway.class);
- TaskExecutorConfiguration taskExecutorConfiguration = mock(TaskExecutorConfiguration.class);
- PowerMockito.when(taskExecutorConfiguration.getNumberOfSlots()).thenReturn(1);
+ TaskManagerConfiguration taskManagerServicesConfiguration = mock(TaskManagerConfiguration.class);
+ PowerMockito.when(taskManagerServicesConfiguration.getNumberSlots()).thenReturn(1);
rpc.registerGateway(resourceManagerAddress, rmGateway);
TaskManagerLocation taskManagerLocation = mock(TaskManagerLocation.class);
@@ -61,12 +62,15 @@ public class TaskExecutorTest extends TestLogger {
NonHaServices haServices = new NonHaServices(resourceManagerAddress);
TaskExecutor taskManager = new TaskExecutor(
- taskExecutorConfiguration,
+ taskManagerServicesConfiguration,
taskManagerLocation,
- rpc, mock(MemoryManager.class),
+ rpc,
+ mock(MemoryManager.class),
mock(IOManager.class),
mock(NetworkEnvironment.class),
- haServices);
+ haServices,
+ mock(MetricRegistry.class),
+ mock(FatalErrorHandler.class));
taskManager.start();
String taskManagerAddress = taskManager.getAddress();
@@ -101,19 +105,22 @@ public class TaskExecutorTest extends TestLogger {
TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServices();
haServices.setResourceManagerLeaderRetriever(testLeaderService);
- TaskExecutorConfiguration taskExecutorConfiguration = mock(TaskExecutorConfiguration.class);
- PowerMockito.when(taskExecutorConfiguration.getNumberOfSlots()).thenReturn(1);
+ TaskManagerConfiguration taskManagerServicesConfiguration = mock(TaskManagerConfiguration.class);
+ PowerMockito.when(taskManagerServicesConfiguration.getNumberSlots()).thenReturn(1);
TaskManagerLocation taskManagerLocation = mock(TaskManagerLocation.class);
when(taskManagerLocation.getResourceID()).thenReturn(resourceID);
TaskExecutor taskManager = new TaskExecutor(
- taskExecutorConfiguration,
+ taskManagerServicesConfiguration,
taskManagerLocation,
- rpc, mock(MemoryManager.class),
+ rpc,
+ mock(MemoryManager.class),
mock(IOManager.class),
mock(NetworkEnvironment.class),
- haServices);
+ haServices,
+ mock(MetricRegistry.class),
+ mock(FatalErrorHandler.class));
taskManager.start();
String taskManagerAddress = taskManager.getAddress();
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
index 1f93e9b..627a25a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
@@ -39,7 +39,6 @@ import org.apache.flink.runtime.io.network.LocalConnectionManager;
import org.apache.flink.runtime.io.network.NetworkEnvironment;
import org.apache.flink.runtime.io.network.TaskEventDispatcher;
import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
-import org.apache.flink.runtime.io.network.netty.NettyConfig;
import org.apache.flink.runtime.io.network.partition.ResultPartitionManager;
import org.apache.flink.runtime.jobmanager.JobManager;
import org.apache.flink.runtime.jobmanager.MemoryArchivist;
@@ -105,7 +104,7 @@ public class TaskManagerComponentsStartupShutdownTest {
final NetworkEnvironmentConfiguration netConf = new NetworkEnvironmentConfiguration(
32, BUFFER_SIZE, MemoryType.HEAP, IOManager.IOMode.SYNC, 0, 0, 0,
- Option.<NettyConfig>empty(), 0, 0);
+ null, 0, 0);
ResourceID taskManagerId = ResourceID.generate();
http://git-wip-us.apache.org/repos/asf/flink/blob/5ca94fb9/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerConfigurationTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerConfigurationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerConfigurationTest.java
index acfbbfd..c0d0455 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerConfigurationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerConfigurationTest.java
@@ -29,7 +29,6 @@ import org.junit.Test;
import scala.Tuple2;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.reflect.Field;
[06/50] [abbrv] flink git commit: [FLINK-4408] [JobManager] Introduce
JobMasterRunner and implement job submission & setting up the ExecutionGraph
Posted by tr...@apache.org.
[FLINK-4408] [JobManager] Introduce JobMasterRunner and implement job submission & setting up the ExecutionGraph
This closes #2480
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/44422f8f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/44422f8f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/44422f8f
Branch: refs/heads/flip-6
Commit: 44422f8fd6b8a30cf200ae87f90b4a3cca836c6d
Parents: 1257037
Author: Kurt Young <yk...@gmail.com>
Authored: Thu Sep 8 12:00:13 2016 +0800
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:27 2016 +0100
----------------------------------------------------------------------
.../flink/api/common/JobExecutionResult.java | 2 +-
.../flink/api/common/JobSubmissionResult.java | 2 +-
.../HighAvailabilityServices.java | 12 +
.../runtime/highavailability/NonHaServices.java | 16 +-
.../runtime/jobmanager/OnCompletionActions.java | 31 ++
.../runtime/jobmanager/scheduler/Scheduler.java | 9 +
.../runtime/jobmaster/JobManagerRunner.java | 288 +++++++++++
.../runtime/jobmaster/JobManagerServices.java | 73 +++
.../flink/runtime/jobmaster/JobMaster.java | 485 ++++++++++++++-----
.../runtime/jobmaster/JobMasterGateway.java | 13 +
.../jobmaster/MiniClusterJobDispatcher.java | 385 +++++++++++++++
.../flink/runtime/rpc/FatalErrorHandler.java | 24 +
.../runtime/taskexecutor/TaskExecutor.java | 12 +
.../TestingHighAvailabilityServices.java | 39 +-
.../jobmaster/JobManagerRunnerMockTest.java | 254 ++++++++++
.../flink/runtime/rpc/RpcConnectionTest.java | 17 +-
16 files changed, 1533 insertions(+), 129 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java b/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
index cb4ecc5..7286cc5 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
@@ -32,7 +32,7 @@ import java.util.concurrent.TimeUnit;
@Public
public class JobExecutionResult extends JobSubmissionResult {
- private long netRuntime;
+ private final long netRuntime;
private final Map<String, Object> accumulatorResults;
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-core/src/main/java/org/apache/flink/api/common/JobSubmissionResult.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/JobSubmissionResult.java b/flink-core/src/main/java/org/apache/flink/api/common/JobSubmissionResult.java
index c5dc869..b0e7e24 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/JobSubmissionResult.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/JobSubmissionResult.java
@@ -26,7 +26,7 @@ import org.apache.flink.annotation.Public;
@Public
public class JobSubmissionResult {
- private JobID jobID;
+ private final JobID jobID;
public JobSubmissionResult(JobID jobID) {
this.jobID = jobID;
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
index 7634176..d67e927 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
@@ -19,6 +19,8 @@
package org.apache.flink.runtime.highavailability;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
+import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
@@ -61,4 +63,14 @@ public interface HighAvailabilityServices {
* @param jobID The identifier of the job running the election.
*/
LeaderElectionService getJobMasterLeaderElectionService(JobID jobID) throws Exception;
+
+ /**
+ * Gets the checkpoint recovery factory for the job manager
+ */
+ CheckpointRecoveryFactory getCheckpointRecoveryFactory() throws Exception;
+
+ /**
+ * Gets the submitted job graph store for the job manager
+ */
+ SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
index 33dc2d7..a2c9cc4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
@@ -19,13 +19,17 @@
package org.apache.flink.runtime.highavailability;
import org.apache.flink.api.common.JobID;
-import org.apache.flink.hadoop.shaded.org.jboss.netty.util.internal.ConcurrentHashMap;
+import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
+import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory;
+import org.apache.flink.runtime.jobmanager.StandaloneSubmittedJobGraphStore;
+import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.leaderelection.StandaloneLeaderElectionService;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -88,4 +92,14 @@ public class NonHaServices implements HighAvailabilityServices {
public LeaderElectionService getJobMasterLeaderElectionService(JobID jobID) throws Exception {
return new StandaloneLeaderElectionService();
}
+
+ @Override
+ public CheckpointRecoveryFactory getCheckpointRecoveryFactory() throws Exception {
+ return new StandaloneCheckpointRecoveryFactory();
+ }
+
+ @Override
+ public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
+ return new StandaloneSubmittedJobGraphStore();
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java
new file mode 100644
index 0000000..6de4253
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmanager;
+
+import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
+
+public interface OnCompletionActions extends FatalErrorHandler {
+
+ void jobFinished(JobExecutionResult result);
+
+ void jobFailed(Throwable cause);
+
+ void jobFinishedByOther();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/Scheduler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/Scheduler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/Scheduler.java
index b839e0e..aa09314 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/Scheduler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/Scheduler.java
@@ -31,6 +31,7 @@ import java.util.Queue;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import akka.dispatch.Futures;
@@ -57,6 +58,7 @@ import org.apache.flink.util.ExceptionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.concurrent.ExecutionContext;
+import scala.concurrent.ExecutionContext$;
/**
* The scheduler is responsible for distributing the ready-to-run tasks among instances and slots.
@@ -110,6 +112,13 @@ public class Scheduler implements InstanceListener, SlotAvailabilityListener, Sl
/**
* Creates a new scheduler.
*/
+ public Scheduler(ExecutorService executor) {
+ this(ExecutionContext$.MODULE$.fromExecutor(executor));
+ }
+
+ /**
+ * Creates a new scheduler.
+ */
public Scheduler(ExecutionContext executionContext) {
this.executionContext = executionContext;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
new file mode 100644
index 0000000..bc2bf9a
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobmanager.OnCompletionActions;
+import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
+import org.apache.flink.runtime.leaderelection.LeaderContender;
+import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.rpc.RpcService;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.UUID;
+import java.util.concurrent.Executor;
+
+/**
+ * The runner for the job manager. It deals with job level leader election and make underlying job manager
+ * properly reacted.
+ */
+public class JobManagerRunner implements LeaderContender, OnCompletionActions {
+
+ private final Logger log = LoggerFactory.getLogger(JobManagerRunner.class);
+
+ /** Lock to ensure that this runner can deal with leader election event and job completion notifies simultaneously */
+ private final Object lock = new Object();
+
+ /** The job graph needs to run */
+ private final JobGraph jobGraph;
+
+ private final OnCompletionActions toNotify;
+
+ /** The execution context which is used to execute futures */
+ private final Executor executionContext;
+
+ // TODO: use this to decide whether the job is finished by other
+ private final CheckpointRecoveryFactory checkpointRecoveryFactory;
+
+ /** Leader election for this job */
+ private final LeaderElectionService leaderElectionService;
+
+ private final JobMaster jobManager;
+
+ /** Leader session id when granted leadership */
+ private UUID leaderSessionID;
+
+ /** flag marking the runner as shut down */
+ private volatile boolean shutdown;
+
+ public JobManagerRunner(
+ final JobGraph jobGraph,
+ final Configuration configuration,
+ final RpcService rpcService,
+ final HighAvailabilityServices haServices,
+ final OnCompletionActions toNotify) throws Exception
+ {
+ this(jobGraph, configuration, rpcService, haServices,
+ JobManagerServices.fromConfiguration(configuration), toNotify);
+ }
+
+ public JobManagerRunner(
+ final JobGraph jobGraph,
+ final Configuration configuration,
+ final RpcService rpcService,
+ final HighAvailabilityServices haServices,
+ final JobManagerServices jobManagerServices,
+ final OnCompletionActions toNotify) throws Exception
+ {
+ this.jobGraph = jobGraph;
+ this.toNotify = toNotify;
+ this.executionContext = rpcService.getExecutor();
+ this.checkpointRecoveryFactory = haServices.getCheckpointRecoveryFactory();
+ this.leaderElectionService = haServices.getJobMasterLeaderElectionService(jobGraph.getJobID());
+ this.leaderSessionID = null;
+
+ this.jobManager = new JobMaster(
+ jobGraph, configuration, rpcService, haServices,
+ jobManagerServices.libraryCacheManager,
+ jobManagerServices.restartStrategyFactory,
+ jobManagerServices.savepointStore,
+ jobManagerServices.timeout,
+ new Scheduler(jobManagerServices.executorService),
+ jobManagerServices.jobManagerMetricGroup,
+ this);
+ }
+
+ //----------------------------------------------------------------------------------------------
+ // Lifecycle management
+ //----------------------------------------------------------------------------------------------
+
+ public void start() throws Exception {
+ jobManager.init();
+ jobManager.start();
+
+ try {
+ leaderElectionService.start(this);
+ }
+ catch (Exception e) {
+ log.error("Could not start the JobManager because the leader election service did not start.", e);
+ throw new Exception("Could not start the leader election service.", e);
+ }
+ }
+
+ public void shutdown() {
+ shutdown(new Exception("The JobManager runner is shutting down"));
+ }
+
+ public void shutdown(Throwable cause) {
+ // TODO what is the cause used for ?
+ shutdownInternally();
+ }
+
+ private void shutdownInternally() {
+ synchronized (lock) {
+ shutdown = true;
+
+ if (leaderElectionService != null) {
+ try {
+ leaderElectionService.stop();
+ } catch (Exception e) {
+ log.error("Could not properly shutdown the leader election service.");
+ }
+ }
+
+ jobManager.shutDown();
+ }
+ }
+
+ //----------------------------------------------------------------------------------------------
+ // Result and error handling methods
+ //----------------------------------------------------------------------------------------------
+
+ /**
+ * Job completion notification triggered by JobManager
+ */
+ @Override
+ public void jobFinished(JobExecutionResult result) {
+ try {
+ shutdownInternally();
+ }
+ finally {
+ if (toNotify != null) {
+ toNotify.jobFinished(result);
+ }
+ }
+ }
+
+ /**
+ * Job completion notification triggered by JobManager
+ */
+ @Override
+ public void jobFailed(Throwable cause) {
+ try {
+ shutdownInternally();
+ }
+ finally {
+ if (toNotify != null) {
+ toNotify.jobFailed(cause);
+ }
+ }
+ }
+
+ /**
+ * Job completion notification triggered by self
+ */
+ @Override
+ public void jobFinishedByOther() {
+ try {
+ shutdownInternally();
+ }
+ finally {
+ if (toNotify != null) {
+ toNotify.jobFinishedByOther();
+ }
+ }
+ }
+
+ /**
+ * Job completion notification triggered by JobManager or self
+ */
+ @Override
+ public void onFatalError(Throwable exception) {
+ // first and in any case, notify our handler, so it can react fast
+ try {
+ if (toNotify != null) {
+ toNotify.onFatalError(exception);
+ }
+ }
+ finally {
+ log.error("JobManager runner encountered a fatal error.", exception);
+ shutdownInternally();
+ }
+ }
+
+ //----------------------------------------------------------------------------------------------
+ // Leadership methods
+ //----------------------------------------------------------------------------------------------
+
+ @Override
+ public void grantLeadership(final UUID leaderSessionID) {
+ synchronized (lock) {
+ if (shutdown) {
+ log.info("JobManagerRunner already shutdown.");
+ return;
+ }
+
+ log.info("JobManager runner for job {} ({}) was granted leadership with session id {} at {}.",
+ jobGraph.getName(), jobGraph.getJobID(), leaderSessionID, getAddress());
+
+ // The operation may be blocking, but since this runner is idle before it been granted the leadership,
+ // it's okay that job manager wait for the operation complete
+ leaderElectionService.confirmLeaderSessionID(leaderSessionID);
+ this.leaderSessionID = leaderSessionID;
+
+ // Double check the leadership after we confirm that, there is a small chance that multiple
+ // job managers schedule the same job after if they try to recover at the same time.
+ // This will eventually be noticed, but can not be ruled out from the beginning.
+ if (leaderElectionService.hasLeadership()) {
+ if (isJobFinishedByOthers()) {
+ log.info("Job {} ({}) already finished by others.", jobGraph.getName(), jobGraph.getJobID());
+ jobFinishedByOther();
+ } else {
+ jobManager.getSelf().startJob();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void revokeLeadership() {
+ synchronized (lock) {
+ if (shutdown) {
+ log.info("JobManagerRunner already shutdown.");
+ return;
+ }
+
+ log.info("JobManager for job {} ({}) was revoked leadership at {}.",
+ jobGraph.getName(), jobGraph.getJobID(), getAddress());
+
+ leaderSessionID = null;
+ jobManager.getSelf().suspendJob(new Exception("JobManager is no longer the leader."));
+ }
+ }
+
+ @Override
+ public String getAddress() {
+ return jobManager.getAddress();
+ }
+
+ @Override
+ public void handleError(Exception exception) {
+ log.error("Leader Election Service encountered a fatal error.", exception);
+ onFatalError(exception);
+ }
+
+ @VisibleForTesting
+ boolean isJobFinishedByOthers() {
+ // TODO
+ return false;
+ }
+
+ @VisibleForTesting
+ boolean isShutdown() {
+ return shutdown;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerServices.java
new file mode 100644
index 0000000..e6beba6
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerServices.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.checkpoint.savepoint.SavepointStore;
+import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager;
+import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory;
+import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup;
+
+import java.util.concurrent.ExecutorService;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Utility class to hold all auxiliary services used by the {@link JobMaster}.
+ */
+public class JobManagerServices {
+
+ public final ExecutorService executorService;
+
+ public final BlobLibraryCacheManager libraryCacheManager;
+
+ public final RestartStrategyFactory restartStrategyFactory;
+
+ public final SavepointStore savepointStore;
+
+ public final Time timeout;
+
+ public final JobManagerMetricGroup jobManagerMetricGroup;
+
+ public JobManagerServices(
+ ExecutorService executorService,
+ BlobLibraryCacheManager libraryCacheManager,
+ RestartStrategyFactory restartStrategyFactory,
+ SavepointStore savepointStore,
+ Time timeout,
+ JobManagerMetricGroup jobManagerMetricGroup) {
+
+ this.executorService = checkNotNull(executorService);
+ this.libraryCacheManager = checkNotNull(libraryCacheManager);
+ this.restartStrategyFactory = checkNotNull(restartStrategyFactory);
+ this.savepointStore = checkNotNull(savepointStore);
+ this.timeout = checkNotNull(timeout);
+ this.jobManagerMetricGroup = checkNotNull(jobManagerMetricGroup);
+ }
+
+ // ------------------------------------------------------------------------
+ // Creating the components from a configuration
+ // ------------------------------------------------------------------------
+
+ public static JobManagerServices fromConfiguration(Configuration config) throws Exception {
+ // TODO not yet implemented
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 1537396..b52a23c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -18,21 +18,50 @@
package org.apache.flink.runtime.jobmaster;
-import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
+import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
+import org.apache.flink.runtime.checkpoint.CompletedCheckpointStore;
+import org.apache.flink.runtime.checkpoint.savepoint.SavepointStore;
+import org.apache.flink.runtime.checkpoint.stats.CheckpointStatsTracker;
+import org.apache.flink.runtime.checkpoint.stats.DisabledCheckpointStatsTracker;
+import org.apache.flink.runtime.checkpoint.stats.SimpleCheckpointStatsTracker;
+import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.client.JobSubmissionException;
+import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager;
+import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+import org.apache.flink.runtime.executiongraph.restart.RestartStrategy;
+import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.leaderelection.LeaderContender;
-import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.jsonplan.JsonPlanGenerator;
+import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobmanager.OnCompletionActions;
+import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
import org.apache.flink.runtime.messages.Acknowledge;
-import org.apache.flink.runtime.rpc.RpcMethod;
+import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup;
import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
import org.apache.flink.runtime.rpc.RpcEndpoint;
+import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
-import org.apache.flink.util.Preconditions;
+import scala.concurrent.ExecutionContext$;
+import scala.concurrent.duration.FiniteDuration;
-import java.util.UUID;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executor;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* JobMaster implementation. The job master is responsible for the execution of a single
@@ -41,7 +70,7 @@ import java.util.UUID;
* It offers the following methods as part of its rpc interface to interact with the JobMaster
* remotely:
* <ul>
- * <li>{@link #updateTaskExecutionState(TaskExecutionState)} updates the task execution state for
+ * <li>{@link #updateTaskExecutionState(TaskExecutionState)} updates the task execution state for
* given task</li>
* </ul>
*/
@@ -52,7 +81,6 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
/** Logical representation of the job */
private final JobGraph jobGraph;
- private final JobID jobID;
/** Configuration of the job */
private final Configuration configuration;
@@ -60,32 +88,67 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
/** Service to contend for and retrieve the leadership of JM and RM */
private final HighAvailabilityServices highAvailabilityServices;
- /** Leader Management */
- private LeaderElectionService leaderElectionService = null;
- private UUID leaderSessionID;
+ /** Blob cache manager used across jobs */
+ private final BlobLibraryCacheManager libraryCacheManager;
+
+ /** Factory to create restart strategy for this job */
+ private final RestartStrategyFactory restartStrategyFactory;
+
+ /** Store for save points */
+ private final SavepointStore savepointStore;
+
+ /** The timeout for this job */
+ private final Time timeout;
+
+ /** The scheduler to use for scheduling new tasks as they are needed */
+ private final Scheduler scheduler;
+
+ /** The metrics group used across jobs */
+ private final JobManagerMetricGroup jobManagerMetricGroup;
+
+ /** The execution context which is used to execute futures */
+ private final Executor executionContext;
+
+ private final OnCompletionActions jobCompletionActions;
+
+ /** The execution graph of this job */
+ private volatile ExecutionGraph executionGraph;
+
+ /** The checkpoint recovery factory used by this job */
+ private CheckpointRecoveryFactory checkpointRecoveryFactory;
+
+ private ClassLoader userCodeLoader;
+
+ private RestartStrategy restartStrategy;
+
+ private MetricGroup jobMetrics;
- /**
- * The JM's Constructor
- *
- * @param jobGraph The representation of the job's execution plan
- * @param configuration The job's configuration
- * @param rpcService The RPC service at which the JM serves
- * @param highAvailabilityService The cluster's HA service from the JM can elect and retrieve leaders.
- */
public JobMaster(
JobGraph jobGraph,
Configuration configuration,
RpcService rpcService,
- HighAvailabilityServices highAvailabilityService) {
-
+ HighAvailabilityServices highAvailabilityService,
+ BlobLibraryCacheManager libraryCacheManager,
+ RestartStrategyFactory restartStrategyFactory,
+ SavepointStore savepointStore,
+ Time timeout,
+ Scheduler scheduler,
+ JobManagerMetricGroup jobManagerMetricGroup,
+ OnCompletionActions jobCompletionActions)
+ {
super(rpcService);
- this.jobGraph = Preconditions.checkNotNull(jobGraph);
- this.jobID = Preconditions.checkNotNull(jobGraph.getJobID());
-
- this.configuration = Preconditions.checkNotNull(configuration);
-
- this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityService);
+ this.jobGraph = checkNotNull(jobGraph);
+ this.configuration = checkNotNull(configuration);
+ this.highAvailabilityServices = checkNotNull(highAvailabilityService);
+ this.libraryCacheManager = checkNotNull(libraryCacheManager);
+ this.restartStrategyFactory = checkNotNull(restartStrategyFactory);
+ this.savepointStore = checkNotNull(savepointStore);
+ this.timeout = checkNotNull(timeout);
+ this.scheduler = checkNotNull(scheduler);
+ this.jobManagerMetricGroup = checkNotNull(jobManagerMetricGroup);
+ this.executionContext = checkNotNull(rpcService.getExecutor());
+ this.jobCompletionActions = checkNotNull(jobCompletionActions);
}
public ResourceManagerGateway getResourceManager() {
@@ -93,93 +156,294 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
}
//----------------------------------------------------------------------------------------------
- // Initialization methods
+ // Lifecycle management
//----------------------------------------------------------------------------------------------
+
+ /**
+ * Initializing the job execution environment, should be called before start. Any error occurred during
+ * initialization will be treated as job submission failure.
+ *
+ * @throws JobSubmissionException
+ */
+ public void init() throws JobSubmissionException {
+ log.info("Initializing job {} ({}).", jobGraph.getName(), jobGraph.getJobID());
+
+ try {
+ // IMPORTANT: We need to make sure that the library registration is the first action,
+ // because this makes sure that the uploaded jar files are removed in case of
+ // unsuccessful
+ try {
+ libraryCacheManager.registerJob(jobGraph.getJobID(), jobGraph.getUserJarBlobKeys(),
+ jobGraph.getClasspaths());
+ } catch (Throwable t) {
+ throw new JobSubmissionException(jobGraph.getJobID(),
+ "Cannot set up the user code libraries: " + t.getMessage(), t);
+ }
+
+ userCodeLoader = libraryCacheManager.getClassLoader(jobGraph.getJobID());
+ if (userCodeLoader == null) {
+ throw new JobSubmissionException(jobGraph.getJobID(),
+ "The user code class loader could not be initialized.");
+ }
+
+ if (jobGraph.getNumberOfVertices() == 0) {
+ throw new JobSubmissionException(jobGraph.getJobID(), "The given job is empty");
+ }
+
+ final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration =
+ jobGraph.getSerializedExecutionConfig()
+ .deserializeValue(userCodeLoader)
+ .getRestartStrategy();
+ if (restartStrategyConfiguration != null) {
+ restartStrategy = RestartStrategyFactory.createRestartStrategy(restartStrategyConfiguration);
+ } else {
+ restartStrategy = restartStrategyFactory.createRestartStrategy();
+ }
+
+ log.info("Using restart strategy {} for {} ({}).", restartStrategy, jobGraph.getName(), jobGraph.getJobID());
+
+ if (jobManagerMetricGroup != null) {
+ jobMetrics = jobManagerMetricGroup.addJob(jobGraph);
+ }
+ if (jobMetrics == null) {
+ jobMetrics = new UnregisteredMetricsGroup();
+ }
+
+ try {
+ checkpointRecoveryFactory = highAvailabilityServices.getCheckpointRecoveryFactory();
+ } catch (Exception e) {
+ log.error("Could not get the checkpoint recovery factory.", e);
+ throw new JobSubmissionException(jobGraph.getJobID(), "Could not get the checkpoint recovery factory.", e);
+ }
+
+ } catch (Throwable t) {
+ log.error("Failed to initializing job {} ({})", jobGraph.getName(), jobGraph.getJobID(), t);
+
+ libraryCacheManager.unregisterJob(jobGraph.getJobID());
+
+ if (t instanceof JobSubmissionException) {
+ throw (JobSubmissionException) t;
+ } else {
+ throw new JobSubmissionException(jobGraph.getJobID(), "Failed to initialize job " +
+ jobGraph.getName() + " (" + jobGraph.getJobID() + ")", t);
+ }
+ }
+ }
+
+ @Override
public void start() {
super.start();
-
- // register at the election once the JM starts
- registerAtElectionService();
}
+ @Override
+ public void shutDown() {
+ super.shutDown();
+
+ suspendJob(new Exception("JobManager is shutting down."));
+ }
//----------------------------------------------------------------------------------------------
- // JobMaster Leadership methods
+ // RPC methods
//----------------------------------------------------------------------------------------------
/**
- * Retrieves the election service and contend for the leadership.
+ * Start to run the job, runtime data structures like ExecutionGraph will be constructed now and checkpoint
+ * being recovered. After this, we will begin to schedule the job.
*/
- private void registerAtElectionService() {
- try {
- leaderElectionService = highAvailabilityServices.getJobMasterLeaderElectionService(jobID);
- leaderElectionService.start(new JobMasterLeaderContender());
- } catch (Exception e) {
- throw new RuntimeException("Fail to register at the election of JobMaster", e);
+ @RpcMethod
+ public void startJob() {
+ log.info("Starting job {} ({}).", jobGraph.getName(), jobGraph.getJobID());
+
+ if (executionGraph != null) {
+ executionGraph = new ExecutionGraph(
+ ExecutionContext$.MODULE$.fromExecutor(executionContext),
+ jobGraph.getJobID(),
+ jobGraph.getName(),
+ jobGraph.getJobConfiguration(),
+ jobGraph.getSerializedExecutionConfig(),
+ new FiniteDuration(timeout.getSize(), timeout.getUnit()),
+ restartStrategy,
+ jobGraph.getUserJarBlobKeys(),
+ jobGraph.getClasspaths(),
+ userCodeLoader,
+ jobMetrics);
+ } else {
+ // TODO: update last active time in JobInfo
}
- }
- /**
- * Start the execution when the leadership is granted.
- *
- * @param newLeaderSessionID The identifier of the new leadership session
- */
- public void grantJobMasterLeadership(final UUID newLeaderSessionID) {
- runAsync(new Runnable() {
- @Override
- public void run() {
- log.info("JobManager {} grants leadership with session id {}.", getAddress(), newLeaderSessionID);
+ try {
+ executionGraph.setScheduleMode(jobGraph.getScheduleMode());
+ executionGraph.setQueuedSchedulingAllowed(jobGraph.getAllowQueuedScheduling());
+
+ try {
+ executionGraph.setJsonPlan(JsonPlanGenerator.generatePlan(jobGraph));
+ } catch (Exception e) {
+ log.warn("Cannot create JSON plan for job {} ({})", jobGraph.getJobID(), jobGraph.getName(), e);
+ executionGraph.setJsonPlan("{}");
+ }
- // The operation may be blocking, but since JM is idle before it grants the leadership, it's okay that
- // JM waits here for the operation's completeness.
- leaderSessionID = newLeaderSessionID;
- leaderElectionService.confirmLeaderSessionID(newLeaderSessionID);
+ // initialize the vertices that have a master initialization hook
+ // file output formats create directories here, input formats create splits
+ if (log.isDebugEnabled()) {
+ log.debug("Running initialization on master for job {} ({}).", jobGraph.getJobID(), jobGraph.getName());
+ }
+ for (JobVertex vertex : jobGraph.getVertices()) {
+ final String executableClass = vertex.getInvokableClassName();
+ if (executableClass == null || executableClass.length() == 0) {
+ throw new JobExecutionException(jobGraph.getJobID(),
+ "The vertex " + vertex.getID() + " (" + vertex.getName() + ") has no invokable class.");
+ }
+ if (vertex.getParallelism() == ExecutionConfig.PARALLELISM_AUTO_MAX) {
+ vertex.setParallelism(scheduler.getTotalNumberOfSlots());
+ }
+
+ try {
+ vertex.initializeOnMaster(userCodeLoader);
+ } catch (Throwable t) {
+ throw new JobExecutionException(jobGraph.getJobID(),
+ "Cannot initialize task '" + vertex.getName() + "': " + t.getMessage(), t);
+ }
+ }
- // TODO:: execute the job when the leadership is granted.
+ // topologically sort the job vertices and attach the graph to the existing one
+ final List<JobVertex> sortedTopology = jobGraph.getVerticesSortedTopologicallyFromSources();
+ if (log.isDebugEnabled()) {
+ log.debug("Adding {} vertices from job graph {} ({}).", sortedTopology.size(),
+ jobGraph.getJobID(), jobGraph.getName());
}
- });
- }
+ executionGraph.attachJobGraph(sortedTopology);
- /**
- * Stop the execution when the leadership is revoked.
- */
- public void revokeJobMasterLeadership() {
- runAsync(new Runnable() {
- @Override
- public void run() {
- log.info("JobManager {} was revoked leadership.", getAddress());
+ if (log.isDebugEnabled()) {
+ log.debug("Successfully created execution graph from job graph {} ({}).",
+ jobGraph.getJobID(), jobGraph.getName());
+ }
- // TODO:: cancel the job's execution and notify all listeners
- cancelAndClearEverything(new Exception("JobManager is no longer the leader."));
+ final JobSnapshottingSettings snapshotSettings = jobGraph.getSnapshotSettings();
+ if (snapshotSettings != null) {
+ List<ExecutionJobVertex> triggerVertices = getExecutionJobVertexWithId(
+ executionGraph, snapshotSettings.getVerticesToTrigger());
+
+ List<ExecutionJobVertex> ackVertices = getExecutionJobVertexWithId(
+ executionGraph, snapshotSettings.getVerticesToAcknowledge());
+
+ List<ExecutionJobVertex> confirmVertices = getExecutionJobVertexWithId(
+ executionGraph, snapshotSettings.getVerticesToConfirm());
+
+ CompletedCheckpointStore completedCheckpoints = checkpointRecoveryFactory.createCheckpointStore(
+ jobGraph.getJobID(), userCodeLoader);
+
+ CheckpointIDCounter checkpointIdCounter = checkpointRecoveryFactory.createCheckpointIDCounter(
+ jobGraph.getJobID());
+
+ // Checkpoint stats tracker
+ boolean isStatsDisabled = configuration.getBoolean(
+ ConfigConstants.JOB_MANAGER_WEB_CHECKPOINTS_DISABLE,
+ ConfigConstants.DEFAULT_JOB_MANAGER_WEB_CHECKPOINTS_DISABLE);
+
+ final CheckpointStatsTracker checkpointStatsTracker;
+ if (isStatsDisabled) {
+ checkpointStatsTracker = new DisabledCheckpointStatsTracker();
+ } else {
+ int historySize = configuration.getInteger(
+ ConfigConstants.JOB_MANAGER_WEB_CHECKPOINTS_HISTORY_SIZE,
+ ConfigConstants.DEFAULT_JOB_MANAGER_WEB_CHECKPOINTS_HISTORY_SIZE);
+ checkpointStatsTracker = new SimpleCheckpointStatsTracker(historySize, ackVertices, jobMetrics);
+ }
+
+ executionGraph.enableSnapshotCheckpointing(
+ snapshotSettings.getCheckpointInterval(),
+ snapshotSettings.getCheckpointTimeout(),
+ snapshotSettings.getMinPauseBetweenCheckpoints(),
+ snapshotSettings.getMaxConcurrentCheckpoints(),
+ triggerVertices,
+ ackVertices,
+ confirmVertices,
+ checkpointIdCounter,
+ completedCheckpoints,
+ savepointStore,
+ checkpointStatsTracker);
+ }
+
+ // TODO: register this class to execution graph as job status change listeners
+
+ // TODO: register client as job / execution status change listeners if they are interested
+
+ /*
+ TODO: decide whether we should take the savepoint before recovery
+
+ if (isRecovery) {
+ // this is a recovery of a master failure (this master takes over)
+ executionGraph.restoreLatestCheckpointedState();
+ } else {
+ if (snapshotSettings != null) {
+ String savepointPath = snapshotSettings.getSavepointPath();
+ if (savepointPath != null) {
+ // got a savepoint
+ log.info("Starting job from savepoint {}.", savepointPath);
+
+ // load the savepoint as a checkpoint into the system
+ final CompletedCheckpoint savepoint = SavepointLoader.loadAndValidateSavepoint(
+ jobGraph.getJobID(), executionGraph.getAllVertices(), savepointStore, savepointPath);
+ executionGraph.getCheckpointCoordinator().getCheckpointStore().addCheckpoint(savepoint);
+
+ // Reset the checkpoint ID counter
+ long nextCheckpointId = savepoint.getCheckpointID() + 1;
+ log.info("Reset the checkpoint ID to " + nextCheckpointId);
+ executionGraph.getCheckpointCoordinator().getCheckpointIdCounter().setCount(nextCheckpointId);
- leaderSessionID = null;
+ executionGraph.restoreLatestCheckpointedState();
+ }
+ }
}
- });
- }
+ */
- /**
- * Handles error occurring in the leader election service
- *
- * @param exception Exception thrown in the leader election service
- */
- public void onJobMasterElectionError(final Exception exception) {
- runAsync(new Runnable() {
- @Override
- public void run() {
- log.error("Received an error from the LeaderElectionService.", exception);
+ } catch (Throwable t) {
+ log.error("Failed to start job {} ({})", jobGraph.getName(), jobGraph.getJobID(), t);
- // TODO:: cancel the job's execution and shutdown the JM
- cancelAndClearEverything(exception);
+ executionGraph.fail(t);
+ executionGraph = null;
- leaderSessionID = null;
+ final Throwable rt;
+ if (t instanceof JobExecutionException) {
+ rt = (JobExecutionException) t;
+ } else {
+ rt = new JobExecutionException(jobGraph.getJobID(),
+ "Failed to start job " + jobGraph.getJobID() + " (" + jobGraph.getName() + ")", t);
}
- });
+ // TODO: notify client about this failure
+
+ jobCompletionActions.jobFailed(rt);
+ return;
+ }
+
+ // start scheduling job in another thread
+ executionContext.execute(new Runnable() {
+ @Override
+ public void run() {
+ if (executionGraph != null) {
+ try {
+ executionGraph.scheduleForExecution(scheduler);
+ } catch (Throwable t) {
+ executionGraph.fail(t);
+ }
+ }
+ }
+ });
}
- //----------------------------------------------------------------------------------------------
- // RPC methods
- //----------------------------------------------------------------------------------------------
+ /**
+ * Suspending job, all the running tasks will be cancelled, and runtime data will be cleared.
+ *
+ * @param cause The reason of why this job been suspended.
+ */
+ @RpcMethod
+ public void suspendJob(final Throwable cause) {
+ if (executionGraph != null) {
+ executionGraph.suspend(cause);
+ executionGraph = null;
+ }
+ }
/**
* Updates the task execution state for a given task.
@@ -208,37 +472,26 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
//----------------------------------------------------------------------------------------------
/**
- * Cancel the current job and notify all listeners the job's cancellation.
+ * Converts JobVertexIDs to corresponding ExecutionJobVertexes
*
- * @param cause Cause for the cancelling.
+ * @param executionGraph The execution graph that holds the relationship
+ * @param vertexIDs The vertexIDs need to be converted
+ * @return The corresponding ExecutionJobVertexes
+ * @throws JobExecutionException
*/
- private void cancelAndClearEverything(Throwable cause) {
- // currently, nothing to do here
- }
-
- // ------------------------------------------------------------------------
- // Utility classes
- // ------------------------------------------------------------------------
- private class JobMasterLeaderContender implements LeaderContender {
-
- @Override
- public void grantLeadership(UUID leaderSessionID) {
- JobMaster.this.grantJobMasterLeadership(leaderSessionID);
- }
-
- @Override
- public void revokeLeadership() {
- JobMaster.this.revokeJobMasterLeadership();
- }
-
- @Override
- public String getAddress() {
- return JobMaster.this.getAddress();
- }
-
- @Override
- public void handleError(Exception exception) {
- onJobMasterElectionError(exception);
+ private static List<ExecutionJobVertex> getExecutionJobVertexWithId(
+ final ExecutionGraph executionGraph, final List<JobVertexID> vertexIDs)
+ throws JobExecutionException
+ {
+ final List<ExecutionJobVertex> ret = new ArrayList<>(vertexIDs.size());
+ for (JobVertexID vertexID : vertexIDs) {
+ final ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(vertexID);
+ if (executionJobVertex == null) {
+ throw new JobExecutionException(executionGraph.getJobID(),
+ "The snapshot checkpointing settings refer to non-existent vertex " + vertexID);
+ }
+ ret.add(executionJobVertex);
}
+ return ret;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
index 86bf17c..b281ea8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
@@ -29,6 +29,19 @@ import org.apache.flink.runtime.taskmanager.TaskExecutionState;
public interface JobMasterGateway extends RpcGateway {
/**
+ * Making this job begins to run.
+ */
+ void startJob();
+
+ /**
+ * Suspending job, all the running tasks will be cancelled, and runtime status will be cleared. Should re-submit
+ * the job before restarting it.
+ *
+ * @param cause The reason of why this job been suspended.
+ */
+ void suspendJob(final Throwable cause);
+
+ /**
* Updates the task execution state for a given task.
*
* @param taskExecutionState New task execution state for a given task
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java
new file mode 100644
index 0000000..792bfd5
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobmanager.OnCompletionActions;
+import org.apache.flink.runtime.rpc.RpcService;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
+/**
+ * The dispatcher that runs in the mini cluster, waits for jobs, and starts job masters
+ * upon receiving jobs.
+ */
+public class MiniClusterJobDispatcher {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MiniClusterJobDispatcher.class);
+
+ // ------------------------------------------------------------------------
+
+ /** lock to ensure that this dispatcher executes only one job at a time */
+ private final Object lock = new Object();
+
+ /** the configuration with which the mini cluster was started */
+ private final Configuration configuration;
+
+ /** the RPC service to use by the job managers */
+ private final RpcService rpcService;
+
+ /** services for discovery, leader election, and recovery */
+ private final HighAvailabilityServices haServices;
+
+ /** al the services that the JobManager needs, such as BLOB service, factories, etc */
+ private final JobManagerServices jobManagerServices;
+
+ /** The number of JobManagers to launch (more than one simulates a high-availability setup) */
+ private final int numJobManagers;
+
+ /** The runner for the job and master. non-null if a job is currently running */
+ private volatile JobManagerRunner[] runners;
+
+ /** flag marking the dispatcher as hut down */
+ private volatile boolean shutdown;
+
+
+ /**
+ * Starts a mini cluster job dispatcher.
+ *
+ * <p>The dispatcher kicks off one JobManager per job, a behavior similar to a
+ * non-highly-available setup.
+ *
+ * @param config The configuration of the mini cluster
+ * @param haServices Access to the discovery, leader election, and recovery services
+ *
+ * @throws Exception Thrown, if the services for the JobMaster could not be started.
+ */
+ public MiniClusterJobDispatcher(
+ Configuration config,
+ RpcService rpcService,
+ HighAvailabilityServices haServices) throws Exception {
+ this(config, rpcService, haServices, 1);
+ }
+
+ /**
+ * Starts a mini cluster job dispatcher.
+ *
+ * <p>The dispatcher may kick off more than one JobManager per job, thus simulating
+ * a highly-available setup.
+ *
+ * @param config The configuration of the mini cluster
+ * @param haServices Access to the discovery, leader election, and recovery services
+ * @param numJobManagers The number of JobMasters to start for each job.
+ *
+ * @throws Exception Thrown, if the services for the JobMaster could not be started.
+ */
+ public MiniClusterJobDispatcher(
+ Configuration config,
+ RpcService rpcService,
+ HighAvailabilityServices haServices,
+ int numJobManagers) throws Exception {
+
+ checkArgument(numJobManagers >= 1);
+ this.configuration = checkNotNull(config);
+ this.rpcService = checkNotNull(rpcService);
+ this.haServices = checkNotNull(haServices);
+ this.numJobManagers = numJobManagers;
+
+ LOG.info("Creating JobMaster services");
+ this.jobManagerServices = JobManagerServices.fromConfiguration(config);
+ }
+
+ // ------------------------------------------------------------------------
+ // life cycle
+ // ------------------------------------------------------------------------
+
+ /**
+ * Shuts down the mini cluster dispatcher. If a job is currently running, that job will be
+ * terminally failed.
+ */
+ public void shutdown() {
+ synchronized (lock) {
+ if (!shutdown) {
+ shutdown = true;
+
+ LOG.info("Shutting down the dispatcher");
+
+ // in this shutdown code we copy the references to the stack first,
+ // to avoid concurrent modification
+
+ JobManagerRunner[] runners = this.runners;
+ if (runners != null) {
+ this.runners = null;
+
+ Exception shutdownException = new Exception("The MiniCluster is shutting down");
+ for (JobManagerRunner runner : runners) {
+ runner.shutdown(shutdownException);
+ }
+ }
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // submitting jobs
+ // ------------------------------------------------------------------------
+
+ /**
+ * This method executes a job in detached mode. The method returns immediately after the job
+ * has been added to the
+ *
+ * @param job The Flink job to execute
+ *
+ * @throws JobExecutionException Thrown if anything went amiss during initial job launch,
+ * or if the job terminally failed.
+ */
+ public void runDetached(JobGraph job) throws JobExecutionException {
+ checkNotNull(job);
+
+ LOG.info("Received job for detached execution {} ({})", job.getName(), job.getJobID());
+
+ synchronized (lock) {
+ checkState(!shutdown, "mini cluster is shut down");
+ checkState(runners == null, "mini cluster can only execute one job at a time");
+
+ OnCompletionActions onJobCompletion = new DetachedFinalizer(numJobManagers);
+
+ this.runners = startJobRunners(job, onJobCompletion);
+ }
+ }
+
+ /**
+ * This method runs a job in blocking mode. The method returns only after the job
+ * completed successfully, or after it failed terminally.
+ *
+ * @param job The Flink job to execute
+ * @return The result of the job execution
+ *
+ * @throws JobExecutionException Thrown if anything went amiss during initial job lauch,
+ * or if the job terminally failed.
+ */
+ public JobExecutionResult runJobBlocking(JobGraph job) throws JobExecutionException, InterruptedException {
+ checkNotNull(job);
+
+ LOG.info("Received job for blocking execution {} ({})", job.getName(), job.getJobID());
+ final BlockingJobSync onJobCompletion = new BlockingJobSync(job.getJobID(), numJobManagers);
+
+ synchronized (lock) {
+ checkState(!shutdown, "mini cluster is shut down");
+ checkState(runners == null, "mini cluster can only execute one job at a time");
+
+ this.runners = startJobRunners(job, onJobCompletion);
+ }
+
+ try {
+ return onJobCompletion.getResult();
+ }
+ finally {
+ // always clear the status for the next job
+ runners = null;
+ }
+ }
+
+ private JobManagerRunner[] startJobRunners(JobGraph job, OnCompletionActions onCompletion) throws JobExecutionException {
+ LOG.info("Starting {} JobMaster(s) for job {} ({})", numJobManagers, job.getName(), job.getJobID());
+
+ JobManagerRunner[] runners = new JobManagerRunner[numJobManagers];
+ for (int i = 0; i < numJobManagers; i++) {
+ try {
+ runners[i] = new JobManagerRunner(job, configuration,
+ rpcService, haServices, jobManagerServices, onCompletion);
+ runners[i].start();
+ }
+ catch (Throwable t) {
+ // shut down all the ones so far
+ Exception shutdownCause = new Exception("Failed to properly start all mini cluster JobManagers", t);
+
+ for (int k = 0; k <= i; k++) {
+ try {
+ if (runners[i] != null) {
+ runners[i].shutdown(shutdownCause);
+ }
+ } catch (Throwable ignored) {
+ // silent shutdown
+ }
+ }
+
+ throw new JobExecutionException(job.getJobID(), "Could not start the JobManager(s) for the job", t);
+ }
+ }
+
+ return runners;
+ }
+
+ // ------------------------------------------------------------------------
+ // test methods to simulate job master failures
+ // ------------------------------------------------------------------------
+
+ public void killJobMaster(int which) {
+ checkArgument(which >= 0 && which < numJobManagers, "no such job master");
+ checkState(!shutdown, "mini cluster is shut down");
+
+ JobManagerRunner[] runners = this.runners;
+ checkState(runners != null, "mini cluster it not executing a job right now");
+
+ runners[which].shutdown(new Throwable("kill JobManager"));
+ }
+
+ // ------------------------------------------------------------------------
+ // utility classes
+ // ------------------------------------------------------------------------
+
+ /**
+ * Simple class that waits for all runners to have reported that they are done.
+ * In the case of a high-availability test setup, there may be multiple runners.
+ * After that, it marks the mini cluster as ready to receive new jobs.
+ */
+ private class DetachedFinalizer implements OnCompletionActions {
+
+ private final AtomicInteger numJobManagersToWaitFor;
+
+ private DetachedFinalizer(int numJobManagersToWaitFor) {
+ this.numJobManagersToWaitFor = new AtomicInteger(numJobManagersToWaitFor);
+ }
+
+ @Override
+ public void jobFinished(JobExecutionResult result) {
+ decrementCheckAndCleanup();
+ }
+
+ @Override
+ public void jobFailed(Throwable cause) {
+ decrementCheckAndCleanup();
+ }
+
+ @Override
+ public void jobFinishedByOther() {
+ decrementCheckAndCleanup();
+ }
+
+ @Override
+ public void onFatalError(Throwable exception) {
+ decrementCheckAndCleanup();
+ }
+
+ private void decrementCheckAndCleanup() {
+ if (numJobManagersToWaitFor.decrementAndGet() == 0) {
+ MiniClusterJobDispatcher.this.runners = null;
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * This class is used to sync on blocking jobs across multiple runners.
+ * Only after all runners reported back that they are finished, the
+ * result will be released.
+ *
+ * That way it is guaranteed that after the blocking job submit call returns,
+ * the dispatcher is immediately free to accept another job.
+ */
+ private static class BlockingJobSync implements OnCompletionActions {
+
+ private final JobID jobId;
+
+ private final CountDownLatch jobMastersToWaitFor;
+
+ private volatile Throwable jobException;
+
+ private volatile Throwable runnerException;
+
+ private volatile JobExecutionResult result;
+
+ BlockingJobSync(JobID jobId, int numJobMastersToWaitFor) {
+ this.jobId = jobId;
+ this.jobMastersToWaitFor = new CountDownLatch(numJobMastersToWaitFor);
+ }
+
+ @Override
+ public void jobFinished(JobExecutionResult jobResult) {
+ this.result = jobResult;
+ jobMastersToWaitFor.countDown();
+ }
+
+ @Override
+ public void jobFailed(Throwable cause) {
+ jobException = cause;
+ jobMastersToWaitFor.countDown();
+ }
+
+ @Override
+ public void jobFinishedByOther() {
+ this.jobMastersToWaitFor.countDown();
+ }
+
+ @Override
+ public void onFatalError(Throwable exception) {
+ if (runnerException == null) {
+ runnerException = exception;
+ }
+ }
+
+ public JobExecutionResult getResult() throws JobExecutionException, InterruptedException {
+ jobMastersToWaitFor.await();
+
+ final Throwable jobFailureCause = this.jobException;
+ final Throwable runnerException = this.runnerException;
+ final JobExecutionResult result = this.result;
+
+ // (1) we check if teh job terminated with an exception
+ // (2) we check whether the job completed successfully
+ // (3) we check if we have exceptions from the JobManagers. the job may still have
+ // completed successfully in that case, if multiple JobMasters were running
+ // and other took over. only if all encounter a fatal error, the job cannot finish
+
+ if (jobFailureCause != null) {
+ if (jobFailureCause instanceof JobExecutionException) {
+ throw (JobExecutionException) jobFailureCause;
+ }
+ else {
+ throw new JobExecutionException(jobId, "The job execution failed", jobFailureCause);
+ }
+ }
+ else if (result != null) {
+ return result;
+ }
+ else if (runnerException != null) {
+ throw new JobExecutionException(jobId,
+ "The job execution failed because all JobManagers encountered fatal errors", runnerException);
+ }
+ else {
+ throw new IllegalStateException("Bug: Job finished with neither error nor result.");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/FatalErrorHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/FatalErrorHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/FatalErrorHandler.java
new file mode 100644
index 0000000..7721117
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/FatalErrorHandler.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rpc;
+
+public interface FatalErrorHandler {
+
+ void onFatalError(Throwable exception);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index cf709c8..9e3c3b9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.taskexecutor;
import akka.actor.ActorSystem;
import com.typesafe.config.Config;
import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.io.network.ConnectionManager;
import org.apache.flink.runtime.io.network.LocalConnectionManager;
@@ -28,6 +29,7 @@ import org.apache.flink.runtime.io.network.TaskEventDispatcher;
import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
import org.apache.flink.runtime.io.network.netty.NettyConnectionManager;
import org.apache.flink.runtime.io.network.partition.ResultPartitionManager;
+import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
import org.apache.flink.runtime.query.KvStateRegistry;
import org.apache.flink.runtime.query.netty.DisabledKvStateRequestStats;
import org.apache.flink.runtime.query.netty.KvStateServer;
@@ -340,6 +342,16 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
public LeaderElectionService getJobMasterLeaderElectionService(JobID jobID) throws Exception {
return null;
}
+
+ @Override
+ public CheckpointRecoveryFactory getCheckpointRecoveryFactory() throws Exception {
+ return null;
+ }
+
+ @Override
+ public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
+ return null;
+ }
};
// start all the TaskManager services (network stack, library cache, ...)
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
index 2ac43be..1a5450d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
@@ -19,10 +19,13 @@
package org.apache.flink.runtime.highavailability;
import org.apache.flink.api.common.JobID;
-import org.apache.flink.hadoop.shaded.org.jboss.netty.util.internal.ConcurrentHashMap;
+import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
+import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import java.util.concurrent.ConcurrentHashMap;
+
/**
* A variant of the HighAvailabilityServices for testing. Each individual service can be set
* to an arbitrary implementation, such as a mock or default service.
@@ -37,6 +40,9 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
private volatile LeaderElectionService resourceManagerLeaderElectionService;
+ private volatile CheckpointRecoveryFactory checkpointRecoveryFactory;
+
+ private volatile SubmittedJobGraphStore submittedJobGraphStore;
// ------------------------------------------------------------------------
// Setters for mock / testing implementations
@@ -58,6 +64,14 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
this.resourceManagerLeaderElectionService = leaderElectionService;
}
+ public void setCheckpointRecoveryFactory(CheckpointRecoveryFactory checkpointRecoveryFactory) {
+ this.checkpointRecoveryFactory = checkpointRecoveryFactory;
+ }
+
+ public void setSubmittedJobGraphStore(SubmittedJobGraphStore submittedJobGraphStore) {
+ this.submittedJobGraphStore = submittedJobGraphStore;
+ }
+
// ------------------------------------------------------------------------
// HA Services Methods
// ------------------------------------------------------------------------
@@ -103,4 +117,27 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
throw new IllegalStateException("ResourceManagerLeaderElectionService has not been set");
}
}
+
+ @Override
+ public CheckpointRecoveryFactory getCheckpointRecoveryFactory() throws Exception {
+ CheckpointRecoveryFactory factory = checkpointRecoveryFactory;
+
+ if (factory != null) {
+ return factory;
+ } else {
+ throw new IllegalStateException("CheckpointRecoveryFactory has not been set");
+ }
+ }
+
+ @Override
+ public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
+ SubmittedJobGraphStore store = submittedJobGraphStore;
+
+ if (store != null) {
+ return store;
+ } else {
+ throw new IllegalStateException("SubmittedJobGraphStore has not been set");
+
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
new file mode 100644
index 0000000..dc3b5fd
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobmanager.OnCompletionActions;
+import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.rpc.RpcService;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.UUID;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(JobManagerRunner.class)
+public class JobManagerRunnerMockTest {
+
+ private JobManagerRunner runner;
+
+ private JobMaster jobManager;
+
+ private JobMasterGateway jobManagerGateway;
+
+ private LeaderElectionService leaderElectionService;
+
+ private TestingOnCompletionActions jobCompletion;
+
+ @Before
+ public void setUp() throws Exception {
+ jobManager = mock(JobMaster.class);
+ jobManagerGateway = mock(JobMasterGateway.class);
+ when(jobManager.getSelf()).thenReturn(jobManagerGateway);
+
+ PowerMockito.whenNew(JobMaster.class).withAnyArguments().thenReturn(jobManager);
+
+ jobCompletion = new TestingOnCompletionActions();
+
+ leaderElectionService = mock(LeaderElectionService.class);
+ when(leaderElectionService.hasLeadership()).thenReturn(true);
+
+ HighAvailabilityServices haServices = mock(HighAvailabilityServices.class);
+ when(haServices.getJobMasterLeaderElectionService(any(JobID.class))).thenReturn(leaderElectionService);
+
+ runner = PowerMockito.spy(new JobManagerRunner(
+ new JobGraph("test"),
+ mock(Configuration.class),
+ mock(RpcService.class),
+ haServices,
+ mock(JobManagerServices.class),
+ jobCompletion));
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ @Test
+ public void testStartAndShutdown() throws Exception {
+ runner.start();
+ verify(jobManager).init();
+ verify(jobManager).start();
+ verify(leaderElectionService).start(runner);
+
+ assertTrue(!jobCompletion.isJobFinished());
+ assertTrue(!jobCompletion.isJobFailed());
+
+ runner.shutdown();
+ verify(leaderElectionService).stop();
+ verify(jobManager).shutDown();
+ }
+
+ @Test
+ public void testShutdownBeforeGrantLeadership() throws Exception {
+ runner.start();
+ verify(jobManager).init();
+ verify(jobManager).start();
+ verify(leaderElectionService).start(runner);
+
+ runner.shutdown();
+ verify(leaderElectionService).stop();
+ verify(jobManager).shutDown();
+
+ assertTrue(!jobCompletion.isJobFinished());
+ assertTrue(!jobCompletion.isJobFailed());
+
+ runner.grantLeadership(UUID.randomUUID());
+ assertTrue(!jobCompletion.isJobFinished());
+ assertTrue(!jobCompletion.isJobFailed());
+
+ }
+
+ @Test
+ public void testJobFinishedByOtherBeforeGrantLeadership() throws Exception {
+ runner.start();
+
+ when(runner.isJobFinishedByOthers()).thenReturn(true);
+ runner.grantLeadership(UUID.randomUUID());
+
+ // runner should shutdown automatic and informed the job completion
+ verify(leaderElectionService).stop();
+ verify(jobManager).shutDown();
+
+ assertTrue(jobCompletion.isJobFinished());
+ assertTrue(jobCompletion.isJobFinishedByOther());
+ }
+
+ @Test
+ public void testJobFinished() throws Exception {
+ runner.start();
+
+ runner.grantLeadership(UUID.randomUUID());
+ verify(jobManagerGateway).startJob();
+ assertTrue(!jobCompletion.isJobFinished());
+
+ // runner been told by JobManager that job is finished
+ runner.jobFinished(mock(JobExecutionResult.class));
+
+ assertTrue(jobCompletion.isJobFinished());
+ assertFalse(jobCompletion.isJobFinishedByOther());
+ verify(leaderElectionService).stop();
+ verify(jobManager).shutDown();
+ assertTrue(runner.isShutdown());
+ }
+
+ @Test
+ public void testJobFailed() throws Exception {
+ runner.start();
+
+ runner.grantLeadership(UUID.randomUUID());
+ verify(jobManagerGateway).startJob();
+ assertTrue(!jobCompletion.isJobFinished());
+
+ // runner been told by JobManager that job is failed
+ runner.jobFailed(new Exception("failed manually"));
+
+ assertTrue(jobCompletion.isJobFailed());
+ verify(leaderElectionService).stop();
+ verify(jobManager).shutDown();
+ assertTrue(runner.isShutdown());
+ }
+
+ @Test
+ public void testLeadershipRevoked() throws Exception {
+ runner.start();
+
+ runner.grantLeadership(UUID.randomUUID());
+ verify(jobManagerGateway).startJob();
+ assertTrue(!jobCompletion.isJobFinished());
+
+ runner.revokeLeadership();
+ verify(jobManagerGateway).suspendJob(any(Throwable.class));
+ assertFalse(runner.isShutdown());
+ }
+
+ @Test
+ public void testRegainLeadership() throws Exception {
+ runner.start();
+
+ runner.grantLeadership(UUID.randomUUID());
+ verify(jobManagerGateway).startJob();
+ assertTrue(!jobCompletion.isJobFinished());
+
+ runner.revokeLeadership();
+ verify(jobManagerGateway).suspendJob(any(Throwable.class));
+ assertFalse(runner.isShutdown());
+
+ runner.grantLeadership(UUID.randomUUID());
+ verify(jobManagerGateway, times(2)).startJob();
+ }
+
+ private static class TestingOnCompletionActions implements OnCompletionActions {
+
+ private volatile JobExecutionResult result;
+
+ private volatile Throwable failedCause;
+
+ private volatile boolean finishedByOther;
+
+ @Override
+ public void jobFinished(JobExecutionResult result) {
+ checkArgument(!isJobFinished(), "job finished already");
+ checkArgument(!isJobFailed(), "job failed already");
+
+ this.result = result;
+ }
+
+ @Override
+ public void jobFailed(Throwable cause) {
+ checkArgument(!isJobFinished(), "job finished already");
+ checkArgument(!isJobFailed(), "job failed already");
+
+ this.failedCause = cause;
+ }
+
+ @Override
+ public void jobFinishedByOther() {
+ checkArgument(!isJobFinished(), "job finished already");
+ checkArgument(!isJobFailed(), "job failed already");
+
+ this.finishedByOther = true;
+ }
+
+ @Override
+ public void onFatalError(Throwable exception) {
+ jobFailed(exception);
+ }
+
+ boolean isJobFinished() {
+ return result != null || finishedByOther;
+ }
+
+ boolean isJobFinishedByOther() {
+ return finishedByOther;
+ }
+
+ boolean isJobFailed() {
+ return failedCause != null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44422f8f/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcConnectionTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcConnectionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcConnectionTest.java
index 6363662..e05c8d8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcConnectionTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcConnectionTest.java
@@ -19,23 +19,21 @@
package org.apache.flink.runtime.rpc;
import akka.actor.ActorSystem;
-import akka.util.Timeout;
+import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
import org.apache.flink.runtime.rpc.exceptions.RpcConnectionException;
import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
-import org.junit.AfterClass;
import org.junit.Test;
import scala.Option;
import scala.Tuple2;
-import scala.concurrent.Await;
-import scala.concurrent.Future;
-import scala.concurrent.duration.FiniteDuration;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -57,19 +55,20 @@ public class RpcConnectionTest {
// we start the RPC service with a very long timeout to ensure that the test
// can only pass if the connection problem is not recognized merely via a timeout
- rpcService = new AkkaRpcService(actorSystem, new Timeout(10000000, TimeUnit.SECONDS));
+ rpcService = new AkkaRpcService(actorSystem, Time.of(10000000, TimeUnit.SECONDS));
Future<TaskExecutorGateway> future = rpcService.connect("foo.bar.com.test.invalid", TaskExecutorGateway.class);
- Await.result(future, new FiniteDuration(10000000, TimeUnit.SECONDS));
+ future.get(10000000, TimeUnit.SECONDS);
fail("should never complete normally");
}
catch (TimeoutException e) {
fail("should not fail with a generic timeout exception");
}
- catch (RpcConnectionException e) {
+ catch (ExecutionException e) {
// that is what we want
- assertTrue("wrong error message", e.getMessage().contains("foo.bar.com.test.invalid"));
+ assertTrue(e.getCause() instanceof RpcConnectionException);
+ assertTrue("wrong error message", e.getCause().getMessage().contains("foo.bar.com.test.invalid"));
}
catch (Throwable t) {
fail("wrong exception: " + t);
[47/50] [abbrv] flink git commit: [FLINK-4851] [rm] Introduce
FatalErrorHandler and MetricRegistry to RM
Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/test/java/org/apache/flink/runtime/util/TestingFatalErrorHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/util/TestingFatalErrorHandler.java b/flink-runtime/src/test/java/org/apache/flink/runtime/util/TestingFatalErrorHandler.java
new file mode 100644
index 0000000..616313c
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/util/TestingFatalErrorHandler.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.util;
+
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Testing fatal error handler which records the occurred exceptions during the execution of the
+ * tests. Captured exceptions are thrown as a {@link TestingException}.
+ */
+public class TestingFatalErrorHandler implements FatalErrorHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(TestingFatalErrorHandler.class);
+ private final AtomicReference<Throwable> atomicThrowable;
+
+ public TestingFatalErrorHandler() {
+ atomicThrowable = new AtomicReference<>(null);
+ }
+
+ public void rethrowError() throws TestingException {
+ Throwable throwable = atomicThrowable.get();
+
+ if (throwable != null) {
+ throw new TestingException(throwable);
+ }
+ }
+
+ public boolean hasExceptionOccurred() {
+ return atomicThrowable.get() != null;
+ }
+
+ public Throwable getException() {
+ return atomicThrowable.get();
+ }
+
+ @Override
+ public void onFatalError(Throwable exception) {
+ LOG.error("OnFatalError:", exception);
+
+ if (!atomicThrowable.compareAndSet(null, exception)) {
+ atomicThrowable.get().addSuppressed(exception);
+ }
+ }
+
+ //------------------------------------------------------------------
+ // static utility classes
+ //------------------------------------------------------------------
+
+ private static final class TestingException extends Exception {
+ public TestingException(String message) {
+ super(message);
+ }
+
+ public TestingException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public TestingException(Throwable cause) {
+ super(cause);
+ }
+
+ private static final long serialVersionUID = -4648195335470914498L;
+ }
+}
[23/50] [abbrv] flink git commit: [FLINK-4738] [TaskManager] Port
TaskManager logic to new Flip-6 TaskManager
Posted by tr...@apache.org.
[FLINK-4738] [TaskManager] Port TaskManager logic to new Flip-6 TaskManager
The ported logic contains the task lifecycle management methods, JobManager association and
setup of TaskManager components.
Introduce Rpc implementations for TaskManager components
Implement metrics setup
Move more TaskManager components out of the constructor to make TaskExecutor more testable
Add RpcMethod annotation to TaskExecutor#confirmCheckpoint
This closes #2594.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/0655adca
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/0655adca
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/0655adca
Branch: refs/heads/flip-6
Commit: 0655adcacc2d4dcd31d104287ed5110687aff1c5
Parents: 1a9e644
Author: Till Rohrmann <tr...@apache.org>
Authored: Wed Sep 28 14:39:51 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:30 2016 +0100
----------------------------------------------------------------------
.../CheckpointCoordinatorGateway.java | 38 ++
.../deployment/TaskDeploymentDescriptor.java | 9 +
.../runtime/executiongraph/PartitionInfo.java | 2 +-
.../flink/runtime/filecache/FileCache.java | 17 +-
.../jobgraph/tasks/InputSplitProvider.java | 3 +-
.../tasks/InputSplitProviderException.java | 36 ++
.../jobmaster/ExecutionGraphException.java | 41 ++
.../runtime/jobmaster/JobManagerException.java | 39 ++
.../flink/runtime/jobmaster/JobMaster.java | 50 +-
.../runtime/jobmaster/JobMasterGateway.java | 30 +-
.../jobmaster/MiniClusterJobDispatcher.java | 2 +-
.../runtime/jobmaster/SerializedInputSplit.java | 39 ++
.../jobmaster/message/NextInputSplit.java | 39 --
.../flink/runtime/operators/DataSourceTask.java | 12 +-
.../runtime/query/KvStateRegistryGateway.java | 57 ++
.../taskexecutor/JobManagerConnection.java | 91 +++
.../runtime/taskexecutor/TaskExecutor.java | 627 +++++++++++++++++--
.../taskexecutor/TaskExecutorGateway.java | 80 ++-
.../taskexecutor/TaskManagerConfiguration.java | 3 +-
.../runtime/taskexecutor/TaskManagerRunner.java | 3 +
.../taskexecutor/TaskManagerServices.java | 51 +-
.../flink/runtime/taskexecutor/TaskSlot.java | 73 +++
.../runtime/taskexecutor/TaskSlotMapping.java | 44 ++
.../exceptions/CheckpointException.java | 41 ++
.../exceptions/PartitionException.java | 41 ++
.../taskexecutor/exceptions/TaskException.java | 41 ++
.../exceptions/TaskManagerException.java | 41 ++
.../exceptions/TaskSubmissionException.java | 41 ++
.../rpc/RpcCheckpointResponder.java | 56 ++
.../taskexecutor/rpc/RpcInputSplitProvider.java | 73 +++
.../rpc/RpcKvStateRegistryListener.java | 73 +++
.../rpc/RpcPartitionStateChecker.java | 48 ++
.../RpcResultPartitionConsumableNotifier.java | 67 ++
.../utils/TaskExecutorMetricsInitializer.java | 257 ++++++++
.../ActorGatewayTaskManagerActions.java | 59 ++
.../ActorGatewayTaskManagerConnection.java | 59 --
.../apache/flink/runtime/taskmanager/Task.java | 23 +-
.../runtime/taskmanager/TaskExecutionState.java | 4 +-
.../taskmanager/TaskInputSplitProvider.java | 49 +-
.../runtime/taskmanager/TaskManagerActions.java | 57 ++
.../taskmanager/TaskManagerConnection.java | 57 --
.../flink/runtime/taskmanager/TaskManager.scala | 12 +-
.../FileCacheDeleteValidationTest.java | 4 +-
.../jobmaster/JobManagerRunnerMockTest.java | 1 -
.../runtime/taskexecutor/TaskExecutorTest.java | 24 +-
.../runtime/taskmanager/TaskAsyncCallTest.java | 2 +-
.../taskmanager/TaskInputSplitProviderTest.java | 3 +-
.../flink/runtime/taskmanager/TaskStopTest.java | 4 +-
.../flink/runtime/taskmanager/TaskTest.java | 4 +-
.../source/InputFormatSourceFunction.java | 8 +-
.../tasks/InterruptSensitiveRestoreTest.java | 11 +-
.../streaming/runtime/tasks/StreamTaskTest.java | 4 +-
52 files changed, 2267 insertions(+), 283 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java
new file mode 100644
index 0000000..5a01e4d
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.rpc.RpcGateway;
+import org.apache.flink.runtime.state.CheckpointStateHandles;
+
+public interface CheckpointCoordinatorGateway extends RpcGateway {
+
+ void acknowledgeCheckpoint(
+ final JobID jobID,
+ final ExecutionAttemptID executionAttemptID,
+ final CheckpointMetaData checkpointInfo,
+ final CheckpointStateHandles checkpointStateHandles);
+
+ void declineCheckpoint(
+ final JobID jobID,
+ final ExecutionAttemptID executionAttemptID,
+ final CheckpointMetaData checkpoint);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
index bf31e51..857628f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
@@ -23,6 +23,7 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.TaskInfo;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.blob.BlobKey;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.state.TaskStateHandles;
@@ -53,6 +54,9 @@ public final class TaskDeploymentDescriptor implements Serializable {
/** The ID referencing the attempt to execute the task. */
private final ExecutionAttemptID executionId;
+ /** The allocation ID of the slot in which the task shall be run */
+ private final AllocationID allocationID;
+
/** The task's name. */
private final String taskName;
@@ -144,6 +148,7 @@ public final class TaskDeploymentDescriptor implements Serializable {
this.requiredClasspaths = checkNotNull(requiredClasspaths);
this.targetSlotNumber = targetSlotNumber;
this.taskStateHandles = taskStateHandles;
+ this.allocationID = new AllocationID();
}
public TaskDeploymentDescriptor(
@@ -306,6 +311,10 @@ public final class TaskDeploymentDescriptor implements Serializable {
return requiredClasspaths;
}
+ public AllocationID getAllocationID() {
+ return allocationID;
+ }
+
@Override
public String toString() {
return String.format("TaskDeploymentDescriptor [job id: %s, job vertex id: %s, " +
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/PartitionInfo.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/PartitionInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/PartitionInfo.java
index 478550c..c816da9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/PartitionInfo.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/PartitionInfo.java
@@ -32,7 +32,7 @@ import java.io.Serializable;
public class PartitionInfo implements Serializable {
private static final long serialVersionUID = 1724490660830968430L;
-
+
private final IntermediateDataSetID intermediateDataSetID;
private final InputChannelDeploymentDescriptor inputChannelDeploymentDescriptor;
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java b/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java
index b5bdcaf..a07f1a0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java
@@ -33,8 +33,6 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry;
import org.apache.flink.api.java.tuple.Tuple4;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.FSDataInputStream;
import org.apache.flink.core.fs.FSDataOutputStream;
import org.apache.flink.core.fs.FileStatus;
@@ -44,6 +42,7 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.util.ExecutorThreadFactory;
import org.apache.flink.util.IOUtils;
+import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -71,17 +70,15 @@ public class FileCache {
// ------------------------------------------------------------------------
- public FileCache(Configuration config) throws IOException {
-
- String tempDirs = config.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH);
+ public FileCache(String[] tempDirectories) throws IOException {
- String[] directories = tempDirs.split(",|" + File.pathSeparator);
- storageDirectories = new File[directories.length];
+ Preconditions.checkNotNull(tempDirectories);
- for (int i = 0; i < directories.length; i++) {
+ storageDirectories = new File[tempDirectories.length];
+
+ for (int i = 0; i < tempDirectories.length; i++) {
String cacheDirName = "flink-dist-cache-" + UUID.randomUUID().toString();
- storageDirectories[i] = new File(directories[i], cacheDirName);
+ storageDirectories[i] = new File(tempDirectories[i], cacheDirName);
String path = storageDirectories[i].getAbsolutePath();
if (storageDirectories[i].mkdirs()) {
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/InputSplitProvider.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/InputSplitProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/InputSplitProvider.java
index e0cde17..464b13f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/InputSplitProvider.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/InputSplitProvider.java
@@ -34,6 +34,7 @@ public interface InputSplitProvider {
* @param userCodeClassLoader used to deserialize input splits
* @return the next input split to be consumed by the calling task or <code>null</code> if the
* task shall not consume any further input splits.
+ * @throws InputSplitProviderException if fetching the next input split fails
*/
- InputSplit getNextInputSplit(ClassLoader userCodeClassLoader);
+ InputSplit getNextInputSplit(ClassLoader userCodeClassLoader) throws InputSplitProviderException;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/InputSplitProviderException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/InputSplitProviderException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/InputSplitProviderException.java
new file mode 100644
index 0000000..ac73c6f
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/InputSplitProviderException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobgraph.tasks;
+
+public class InputSplitProviderException extends Exception {
+
+ private static final long serialVersionUID = -8043190713983651548L;
+
+ public InputSplitProviderException(String message) {
+ super(message);
+ }
+
+ public InputSplitProviderException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public InputSplitProviderException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/ExecutionGraphException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/ExecutionGraphException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/ExecutionGraphException.java
new file mode 100644
index 0000000..7c35f3d
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/ExecutionGraphException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+
+/**
+ * Exceptions thrown by operations on the {@link ExecutionGraph} by the {@link JobMaster}.
+ */
+public class ExecutionGraphException extends JobManagerException {
+
+ private static final long serialVersionUID = -5439002256464886357L;
+
+ public ExecutionGraphException(String message) {
+ super(message);
+ }
+
+ public ExecutionGraphException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ExecutionGraphException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerException.java
new file mode 100644
index 0000000..bc2759d
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+/**
+ * Base exception thrown by the {@link JobMaster}.
+ */
+public class JobManagerException extends Exception {
+
+ private static final long serialVersionUID = -7290962952242188064L;
+
+ public JobManagerException(final String message) {
+ super(message);
+ }
+
+ public JobManagerException(final String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public JobManagerException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index e67a167..8f3a342 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -38,6 +38,7 @@ import org.apache.flink.runtime.checkpoint.stats.SimpleCheckpointStatsTracker;
import org.apache.flink.runtime.client.JobExecutionException;
import org.apache.flink.runtime.client.JobSubmissionException;
import org.apache.flink.runtime.client.SerializedJobExecutionResult;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager;
@@ -60,9 +61,9 @@ import org.apache.flink.runtime.jobgraph.jsonplan.JsonPlanGenerator;
import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
import org.apache.flink.runtime.jobmanager.OnCompletionActions;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
-import org.apache.flink.runtime.jobmaster.message.NextInputSplit;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup;
import org.apache.flink.runtime.registration.RegisteredRpcConnection;
import org.apache.flink.runtime.registration.RegistrationResponse;
@@ -71,6 +72,7 @@ import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.runtime.state.CheckpointStateHandles;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import org.apache.flink.runtime.util.SerializedThrowable;
import org.apache.flink.util.InstantiationUtil;
@@ -507,12 +509,18 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
* @return Acknowledge the task execution state update
*/
@RpcMethod
- public boolean updateTaskExecutionState(final TaskExecutionState taskExecutionState) {
+ public Acknowledge updateTaskExecutionState(final TaskExecutionState taskExecutionState) throws ExecutionGraphException {
if (taskExecutionState == null) {
- return false;
+ throw new NullPointerException("TaskExecutionState must not be null.");
+ }
+
+ if (executionGraph.updateState(taskExecutionState)) {
+ return Acknowledge.get();
} else {
- return executionGraph.updateState(taskExecutionState);
+ throw new ExecutionGraphException("The execution attempt " +
+ taskExecutionState.getID() + " was not found.");
}
+
}
//----------------------------------------------------------------------------------------------\u2028
@@ -531,7 +539,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
}
@RpcMethod
- public NextInputSplit requestNextInputSplit(
+ public SerializedInputSplit requestNextInputSplit(
final JobVertexID vertexID,
final ExecutionAttemptID executionAttempt) throws Exception
{
@@ -569,7 +577,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
try {
final byte[] serializedInputSplit = InstantiationUtil.serializeObject(nextInputSplit);
- return new NextInputSplit(serializedInputSplit);
+ return new SerializedInputSplit(serializedInputSplit);
} catch (Exception ex) {
log.error("Could not serialize the next input split of class {}.", nextInputSplit.getClass(), ex);
IOException reason = new IOException("Could not serialize the next input split of class " +
@@ -591,8 +599,36 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
}
@RpcMethod
- public void scheduleOrUpdateConsumers(final ResultPartitionID partitionID) {
+ public Acknowledge scheduleOrUpdateConsumers(final ResultPartitionID partitionID) {
executionGraph.scheduleOrUpdateConsumers(partitionID);
+ return Acknowledge.get();
+ }
+
+ @RpcMethod
+ public void disconnectTaskManager(final ResourceID resourceID) {
+ throw new UnsupportedOperationException();
+ }
+
+ @RpcMethod
+ public void acknowledgeCheckpoint(
+ JobID jobID,
+ ExecutionAttemptID executionAttemptID,
+ long checkpointID,
+ CheckpointStateHandles checkpointStateHandles,
+ long synchronousDurationMillis,
+ long asynchronousDurationMillis,
+ long bytesBufferedInAlignment,
+ long alignmentDurationNanos) {
+ throw new UnsupportedOperationException();
+ }
+
+ @RpcMethod
+ public void declineCheckpoint(
+ JobID jobID,
+ ExecutionAttemptID executionAttemptID,
+ long checkpointID,
+ long checkpointTimestamp) {
+ throw new UnsupportedOperationException();
}
//----------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
index 686a3f3..e3e57d4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
@@ -18,6 +18,9 @@
package org.apache.flink.runtime.jobmaster;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.checkpoint.CheckpointCoordinatorGateway;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
@@ -27,8 +30,8 @@ import org.apache.flink.runtime.io.network.partition.ResultPartition;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.jobmaster.message.NextInputSplit;
-import org.apache.flink.runtime.rpc.RpcGateway;
+import org.apache.flink.runtime.messages.Acknowledge;
+import org.apache.flink.runtime.rpc.RpcTimeout;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import java.util.UUID;
@@ -36,7 +39,7 @@ import java.util.UUID;
/**
* {@link JobMaster} rpc gateway interface
*/
-public interface JobMasterGateway extends RpcGateway {
+public interface JobMasterGateway extends CheckpointCoordinatorGateway {
/**
* Starting the job under the given leader session ID.
@@ -57,20 +60,19 @@ public interface JobMasterGateway extends RpcGateway {
* @param taskExecutionState New task execution state for a given task
* @return Future flag of the task execution state update result
*/
- Future<Boolean> updateTaskExecutionState(TaskExecutionState taskExecutionState);
+ Future<Acknowledge> updateTaskExecutionState(TaskExecutionState taskExecutionState);
/**
* Requesting next input split for the {@link ExecutionJobVertex}. The next input split is sent back to the sender
- * as a {@link NextInputSplit} message.
+ * as a {@link SerializedInputSplit} message.
*
* @param vertexID The job vertex id
* @param executionAttempt The execution attempt id
* @return The future of the input split. If there is no further input split, will return an empty object.
- * @throws Exception if some error occurred or information mismatch.
*/
- Future<NextInputSplit> requestNextInputSplit(
+ Future<SerializedInputSplit> requestNextInputSplit(
final JobVertexID vertexID,
- final ExecutionAttemptID executionAttempt) throws Exception;
+ final ExecutionAttemptID executionAttempt);
/**
* Requests the current state of the partition.
@@ -96,6 +98,16 @@ public interface JobMasterGateway extends RpcGateway {
* The JobManager then can decide when to schedule the partition consumers of the given session.
*
* @param partitionID The partition which has already produced data
+ * @param timeout before the rpc call fails
+ * @return Future acknowledge of the schedule or update operation
*/
- void scheduleOrUpdateConsumers(final ResultPartitionID partitionID);
+ Future<Acknowledge> scheduleOrUpdateConsumers(final ResultPartitionID partitionID, @RpcTimeout final Time timeout);
+
+ /**
+ * Disconnects the given {@link org.apache.flink.runtime.taskexecutor.TaskExecutor} from the
+ * {@link JobMaster}.
+ *
+ * @param resourceID identifying the TaskManager to disconnect
+ */
+ void disconnectTaskManager(ResourceID resourceID);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java
index 792bfd5..e8fb5bb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java
@@ -356,7 +356,7 @@ public class MiniClusterJobDispatcher {
final Throwable runnerException = this.runnerException;
final JobExecutionResult result = this.result;
- // (1) we check if teh job terminated with an exception
+ // (1) we check if the job terminated with an exception
// (2) we check whether the job completed successfully
// (3) we check if we have exceptions from the JobManagers. the job may still have
// completed successfully in that case, if multiple JobMasters were running
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/SerializedInputSplit.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/SerializedInputSplit.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/SerializedInputSplit.java
new file mode 100644
index 0000000..bfdc65a
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/SerializedInputSplit.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+import java.io.Serializable;
+
+public class SerializedInputSplit implements Serializable {
+ private static final long serialVersionUID = -2063021844254152064L;
+
+ private final byte[] inputSplitData;
+
+ public SerializedInputSplit(byte[] inputSplitData) {
+ this.inputSplitData = inputSplitData;
+ }
+
+ public byte[] getInputSplitData() {
+ return inputSplitData;
+ }
+
+ public boolean isEmpty() {
+ return inputSplitData == null || inputSplitData.length == 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/NextInputSplit.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/NextInputSplit.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/NextInputSplit.java
deleted file mode 100644
index fe511ed..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/NextInputSplit.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.jobmaster.message;
-
-import java.io.Serializable;
-
-/**
- * Contains the next input split for a task.
- */
-public class NextInputSplit implements Serializable {
-
- private static final long serialVersionUID = -1355784074565856240L;
-
- private final byte[] splitData;
-
- public NextInputSplit(final byte[] splitData) {
- this.splitData = splitData;
- }
-
- public byte[] getSplitData() {
- return splitData;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
index 39d3453..2600b2c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
@@ -32,6 +32,7 @@ import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
+import org.apache.flink.runtime.jobgraph.tasks.InputSplitProviderException;
import org.apache.flink.runtime.metrics.groups.OperatorMetricGroup;
import org.apache.flink.runtime.operators.chaining.ChainedDriver;
import org.apache.flink.runtime.operators.chaining.ExceptionInChainedStubException;
@@ -333,9 +334,14 @@ public class DataSourceTask<OT> extends AbstractInvokable {
if (nextSplit != null) {
return true;
}
-
- InputSplit split = provider.getNextInputSplit(getUserCodeClassLoader());
-
+
+ final InputSplit split;
+ try {
+ split = provider.getNextInputSplit(getUserCodeClassLoader());
+ } catch (InputSplitProviderException e) {
+ throw new RuntimeException("Could not retrieve next input split.", e);
+ }
+
if (split != null) {
this.nextSplit = split;
return true;
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/query/KvStateRegistryGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/query/KvStateRegistryGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/query/KvStateRegistryGateway.java
new file mode 100644
index 0000000..d285074
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/query/KvStateRegistryGateway.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.query;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.rpc.RpcGateway;
+import org.apache.flink.runtime.state.KeyGroupRange;
+
+public interface KvStateRegistryGateway extends RpcGateway {
+ /**
+ * Notifies the listener about a registered KvState instance.
+ *
+ * @param jobId Job ID the KvState instance belongs to
+ * @param jobVertexId JobVertexID the KvState instance belongs to
+ * @param keyGroupRange Key group range the KvState instance belongs to
+ * @param registrationName Name under which the KvState is registered
+ * @param kvStateId ID of the KvState instance
+ */
+ void notifyKvStateRegistered(
+ JobID jobId,
+ JobVertexID jobVertexId,
+ KeyGroupRange keyGroupRange,
+ String registrationName,
+ KvStateID kvStateId,
+ KvStateServerAddress kvStateServerAddress);
+
+ /**
+ * Notifies the listener about an unregistered KvState instance.
+ *
+ * @param jobId Job ID the KvState instance belongs to
+ * @param jobVertexId JobVertexID the KvState instance belongs to
+ * @param keyGroupRange Key group range the KvState instance belongs to
+ * @param registrationName Name under which the KvState is registered
+ */
+ void notifyKvStateUnregistered(
+ JobID jobId,
+ JobVertexID jobVertexId,
+ KeyGroupRange keyGroupRange,
+ String registrationName);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerConnection.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerConnection.java
new file mode 100644
index 0000000..ef62ef1
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerConnection.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor;
+
+import org.apache.flink.runtime.execution.librarycache.LibraryCacheManager;
+import org.apache.flink.runtime.io.network.netty.PartitionStateChecker;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionConsumableNotifier;
+import org.apache.flink.runtime.jobmaster.JobMasterGateway;
+import org.apache.flink.runtime.taskmanager.CheckpointResponder;
+import org.apache.flink.runtime.taskmanager.TaskManagerActions;
+import org.apache.flink.util.Preconditions;
+
+/**
+ * Container class for JobManager specific communication utils used by the {@link TaskExecutor}.
+ */
+public class JobManagerConnection {
+
+ // Gateway to the job master
+ private final JobMasterGateway jobMasterGateway;
+
+ // Task manager actions with respect to the connected job manager
+ private final TaskManagerActions taskManagerActions;
+
+ // Checkpoint responder for the specific job manager
+ private final CheckpointResponder checkpointResponder;
+
+ // Library cache manager connected to the specific job manager
+ private final LibraryCacheManager libraryCacheManager;
+
+ // Result partition consumable notifier for the specific job manager
+ private final ResultPartitionConsumableNotifier resultPartitionConsumableNotifier;
+
+ // Partition state checker for the specific job manager
+ private final PartitionStateChecker partitionStateChecker;
+
+ public JobManagerConnection(
+ JobMasterGateway jobMasterGateway,
+ TaskManagerActions taskManagerActions,
+ CheckpointResponder checkpointResponder,
+ LibraryCacheManager libraryCacheManager,
+ ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
+ PartitionStateChecker partitionStateChecker) {
+
+ this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
+ this.taskManagerActions = Preconditions.checkNotNull(taskManagerActions);
+ this.checkpointResponder = Preconditions.checkNotNull(checkpointResponder);
+ this.libraryCacheManager = Preconditions.checkNotNull(libraryCacheManager);
+ this.resultPartitionConsumableNotifier = Preconditions.checkNotNull(resultPartitionConsumableNotifier);
+ this.partitionStateChecker = Preconditions.checkNotNull(partitionStateChecker);
+ }
+
+ public JobMasterGateway getJobManagerGateway() {
+ return jobMasterGateway;
+ }
+
+ public TaskManagerActions getTaskManagerActions() {
+ return taskManagerActions;
+ }
+
+ public CheckpointResponder getCheckpointResponder() {
+ return checkpointResponder;
+ }
+
+ public LibraryCacheManager getLibraryCacheManager() {
+ return libraryCacheManager;
+ }
+
+ public ResultPartitionConsumableNotifier getResultPartitionConsumableNotifier() {
+ return resultPartitionConsumableNotifier;
+ }
+
+ public PartitionStateChecker getPartitionStateChecker() {
+ return partitionStateChecker;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index c0041a3..35b639b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -18,15 +18,48 @@
package org.apache.flink.runtime.taskexecutor;
+import org.apache.flink.configuration.UnmodifiableConfiguration;
+import org.apache.flink.runtime.accumulators.AccumulatorSnapshot;
+import org.apache.flink.runtime.blob.BlobCache;
+import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.clusterframework.types.SlotID;
import org.apache.flink.runtime.instance.InstanceID;
+import org.apache.flink.runtime.concurrent.ApplyFunction;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
+import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager;
+import org.apache.flink.runtime.execution.librarycache.LibraryCacheManager;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.executiongraph.PartitionInfo;
+import org.apache.flink.runtime.filecache.FileCache;
+import org.apache.flink.runtime.io.network.netty.PartitionStateChecker;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionConsumableNotifier;
+import org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate;
+import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
+import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
+import org.apache.flink.runtime.jobmaster.JobMasterGateway;
+import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRegistered;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRejected;
+import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
+import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
import org.apache.flink.runtime.rpc.FatalErrorHandler;
+import org.apache.flink.runtime.taskexecutor.exceptions.CheckpointException;
+import org.apache.flink.runtime.taskexecutor.exceptions.PartitionException;
+import org.apache.flink.runtime.taskexecutor.exceptions.TaskException;
+import org.apache.flink.runtime.taskexecutor.exceptions.TaskSubmissionException;
+import org.apache.flink.runtime.taskexecutor.rpc.RpcCheckpointResponder;
+import org.apache.flink.runtime.taskexecutor.rpc.RpcInputSplitProvider;
+import org.apache.flink.runtime.taskexecutor.rpc.RpcPartitionStateChecker;
+import org.apache.flink.runtime.taskexecutor.rpc.RpcResultPartitionConsumableNotifier;
+import org.apache.flink.runtime.taskmanager.CheckpointResponder;
+import org.apache.flink.runtime.taskmanager.Task;
+import org.apache.flink.runtime.taskmanager.TaskExecutionState;
+import org.apache.flink.runtime.taskmanager.TaskManagerActions;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
@@ -38,11 +71,17 @@ import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.util.Preconditions;
import java.util.HashSet;
import java.util.Set;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
import java.util.UUID;
import static org.apache.flink.util.Preconditions.checkArgument;
@@ -50,12 +89,10 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* TaskExecutor implementation. The task executor is responsible for the execution of multiple
- * {@link org.apache.flink.runtime.taskmanager.Task}.
+ * {@link Task}.
*/
public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
- private static final Logger LOG = LoggerFactory.getLogger(TaskExecutor.class);
-
/** The connection information of this task manager */
private final TaskManagerLocation taskManagerLocation;
@@ -77,19 +114,38 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
/** The metric registry in the task manager */
private final MetricRegistry metricRegistry;
- /** The number of slots in the task manager, should be 1 for YARN */
- private final int numberOfSlots;
-
/** The fatal error handler to use in case of a fatal error */
private final FatalErrorHandler fatalErrorHandler;
+ private final TaskManagerMetricGroup taskManagerMetricGroup;
+
+ private final BroadcastVariableManager broadcastVariableManager;
+
/** Slots which have become available but haven't been confirmed by the RM */
private final Set<SlotID> unconfirmedFreeSlots;
+
+ private final FileCache fileCache;
+
+ // TODO: Try to get rid of it
+ private final TaskManagerRuntimeInfo taskManagerRuntimeInfo;
+
// --------- resource manager --------
private TaskExecutorToResourceManagerConnection resourceManagerConnection;
+ // --------- job manager connections -----------
+
+ private Map<ResourceID, JobManagerConnection> jobManagerConnections;
+
+ // --------- Slot allocation table --------
+
+ private Map<AllocationID, TaskSlot> taskSlots;
+
+ // --------- Slot allocation table --------
+
+ private Map<ExecutionAttemptID, TaskSlotMapping> taskSlotMappings;
+
// ------------------------------------------------------------------------
public TaskExecutor(
@@ -101,6 +157,9 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
NetworkEnvironment networkEnvironment,
HighAvailabilityServices haServices,
MetricRegistry metricRegistry,
+ TaskManagerMetricGroup taskManagerMetricGroup,
+ BroadcastVariableManager broadcastVariableManager,
+ FileCache fileCache,
FatalErrorHandler fatalErrorHandler) {
super(rpcService);
@@ -115,10 +174,19 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
this.haServices = checkNotNull(haServices);
this.metricRegistry = checkNotNull(metricRegistry);
this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
+ this.taskManagerMetricGroup = checkNotNull(taskManagerMetricGroup);
+ this.broadcastVariableManager = checkNotNull(broadcastVariableManager);
+ this.fileCache = checkNotNull(fileCache);
+ this.taskManagerRuntimeInfo = new TaskManagerRuntimeInfo(
+ taskManagerLocation.getHostname(),
+ new UnmodifiableConfiguration(taskManagerConfiguration.getConfiguration()),
+ taskManagerConfiguration.getTmpDirPaths());
- this.numberOfSlots = taskManagerConfiguration.getNumberSlots();
+ this.jobManagerConnections = new HashMap<>(4);
this.unconfirmedFreeSlots = new HashSet<>();
+ this.taskSlots = new HashMap<>(taskManagerConfiguration.getNumberSlots());
+ this.taskSlotMappings = new HashMap<>(taskManagerConfiguration.getNumberSlots() * 2);
}
// ------------------------------------------------------------------------
@@ -137,12 +205,436 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
+ /**
+ * Called to shut down the TaskManager. The method closes all TaskManager services.
+ */
+ @Override
+ public void shutDown() {
+ log.info("Stopping TaskManager {}.", getAddress());
+
+ if (resourceManagerConnection.isConnected()) {
+ try {
+ resourceManagerConnection.close();
+ } catch (Exception e) {
+ log.error("Could not cleanly close the ResourceManager connection.", e);
+ }
+ }
+
+ try {
+ ioManager.shutdown();
+ } catch (Exception e) {
+ log.error("IOManager did not shut down properly.", e);
+ }
+
+ try {
+ memoryManager.shutdown();
+ } catch (Exception e) {
+ log.error("MemoryManager did not shut down properly.", e);
+ }
+
+ try {
+ networkEnvironment.shutdown();
+ } catch (Exception e) {
+ log.error("Network environment did not shut down properly.", e);
+ }
+
+ try {
+ fileCache.shutdown();
+ } catch (Exception e) {
+ log.error("File cache did not shut down properly.", e);
+ }
+
+ try {
+ metricRegistry.shutdown();
+ } catch (Exception e) {
+ log.error("MetricRegistry did not shut down properly.", e);
+ }
+
+ log.info("Stopped TaskManager {}.", getAddress());
+ }
+
+ // ========================================================================
+ // RPC methods
+ // ========================================================================
+
+ // ----------------------------------------------------------------------
+ // Task lifecycle RPCs
+ // ----------------------------------------------------------------------
+
+ @RpcMethod
+ public Acknowledge submitTask(TaskDeploymentDescriptor tdd, ResourceID jobManagerID) throws TaskSubmissionException {
+
+ JobManagerConnection jobManagerConnection = getJobManagerConnection(jobManagerID);
+
+ if (jobManagerConnection == null) {
+ final String message = "Could not submit task because JobManager " + jobManagerID +
+ " was not associated.";
+
+ log.debug(message);
+ throw new TaskSubmissionException(message);
+ }
+
+ TaskSlot taskSlot = taskSlots.get(tdd.getAllocationID());
+
+ if (taskSlot == null) {
+ final String message = "No task slot allocated for allocation ID " + tdd.getAllocationID() + '.';
+ log.debug(message);
+ throw new TaskSubmissionException(message);
+ }
+
+ TaskMetricGroup taskMetricGroup = taskManagerMetricGroup.addTaskForJob(tdd);
+
+ InputSplitProvider inputSplitProvider = new RpcInputSplitProvider(
+ jobManagerConnection.getJobManagerGateway(),
+ tdd.getJobID(),
+ tdd.getVertexID(),
+ tdd.getExecutionId(),
+ taskManagerConfiguration.getTimeout());
+
+ TaskManagerActions taskManagerActions = jobManagerConnection.getTaskManagerActions();
+ CheckpointResponder checkpointResponder = jobManagerConnection.getCheckpointResponder();
+ LibraryCacheManager libraryCache = jobManagerConnection.getLibraryCacheManager();
+ ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = jobManagerConnection.getResultPartitionConsumableNotifier();
+ PartitionStateChecker partitionStateChecker = jobManagerConnection.getPartitionStateChecker();
+
+ Task task = new Task(
+ tdd,
+ memoryManager,
+ ioManager,
+ networkEnvironment,
+ broadcastVariableManager,
+ taskManagerActions,
+ inputSplitProvider,
+ checkpointResponder,
+ libraryCache,
+ fileCache,
+ taskManagerRuntimeInfo,
+ taskMetricGroup,
+ resultPartitionConsumableNotifier,
+ partitionStateChecker,
+ getRpcService().getExecutor());
+
+ log.info("Received task {}.", task.getTaskInfo().getTaskNameWithSubtasks());
+
+ if(taskSlot.add(task)) {
+ TaskSlotMapping taskSlotMapping = new TaskSlotMapping(task, taskSlot);
+
+ taskSlotMappings.put(task.getExecutionId(), taskSlotMapping);
+ task.startTaskThread();
+
+ return Acknowledge.get();
+ } else {
+ final String message = "TaskManager already contains a task for id " +
+ task.getExecutionId() + '.';
+
+ log.debug(message);
+ throw new TaskSubmissionException(message);
+ }
+ }
+
+ @RpcMethod
+ public Acknowledge cancelTask(ExecutionAttemptID executionAttemptID) throws TaskException {
+ final Task task = getTask(executionAttemptID);
+
+ if (task != null) {
+ try {
+ task.cancelExecution();
+ return Acknowledge.get();
+ } catch (Throwable t) {
+ throw new TaskException("Cannot cancel task for execution " + executionAttemptID + '.', t);
+ }
+ } else {
+ final String message = "Cannot find task to stop for execution " + executionAttemptID + '.';
+
+ log.debug(message);
+ throw new TaskException(message);
+ }
+ }
+
+ @RpcMethod
+ public Acknowledge stopTask(ExecutionAttemptID executionAttemptID) throws TaskException {
+ final Task task = getTask(executionAttemptID);
+
+ if (task != null) {
+ try {
+ task.stopExecution();
+ return Acknowledge.get();
+ } catch (Throwable t) {
+ throw new TaskException("Cannot stop task for execution " + executionAttemptID + '.', t);
+ }
+ } else {
+ final String message = "Cannot find task to stop for execution " + executionAttemptID + '.';
+
+ log.debug(message);
+ throw new TaskException(message);
+ }
+ }
+
+ // ----------------------------------------------------------------------
+ // Partition lifecycle RPCs
+ // ----------------------------------------------------------------------
+
+ @RpcMethod
+ public Acknowledge updatePartitions(final ExecutionAttemptID executionAttemptID, Collection<PartitionInfo> partitionInfos) throws PartitionException {
+ final Task task = getTask(executionAttemptID);
+
+ if (task != null) {
+ for (final PartitionInfo partitionInfo: partitionInfos) {
+ IntermediateDataSetID intermediateResultPartitionID = partitionInfo.getIntermediateDataSetID();
+
+ final SingleInputGate singleInputGate = task.getInputGateById(intermediateResultPartitionID);
+
+ if (singleInputGate != null) {
+ // Run asynchronously because it might be blocking
+ getRpcService().execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ singleInputGate.updateInputChannel(partitionInfo.getInputChannelDeploymentDescriptor());
+ } catch (IOException | InterruptedException e) {
+ log.error("Could not update input data location for task {}. Trying to fail task.", task.getTaskInfo().getTaskName(), e);
+
+ try {
+ task.failExternally(e);
+ } catch (RuntimeException re) {
+ // TODO: Check whether we need this or make exception in failExtenally checked
+ log.error("Failed canceling task with execution ID {} after task update failure.", executionAttemptID, re);
+ }
+ }
+ }
+ });
+ } else {
+ throw new PartitionException("No reader with ID " +
+ intermediateResultPartitionID + " for task " + executionAttemptID +
+ " was found.");
+ }
+ }
+
+ return Acknowledge.get();
+ } else {
+ log.debug("Discard update for input partitions of task {}. Task is no longer running.", executionAttemptID);
+ return Acknowledge.get();
+ }
+ }
+
+ @RpcMethod
+ public void failPartition(ExecutionAttemptID executionAttemptID) {
+ log.info("Discarding the results produced by task execution {}.", executionAttemptID);
+
+ try {
+ networkEnvironment.getResultPartitionManager().releasePartitionsProducedBy(executionAttemptID);
+ } catch (Throwable t) {
+ // TODO: Do we still need this catch branch?
+ onFatalError(t);
+ }
+
+ // TODO: Maybe it's better to return an Acknowledge here to notify the JM about the success/failure with an Exception
+ }
+
+ // ----------------------------------------------------------------------
+ // Checkpointing RPCs
+ // ----------------------------------------------------------------------
+
+ @RpcMethod
+ public Acknowledge triggerCheckpoint(ExecutionAttemptID executionAttemptID, long checkpointId, long checkpointTimestamp) throws CheckpointException {
+ log.debug("Trigger checkpoint {}@{} for {}.", checkpointId, checkpointTimestamp, executionAttemptID);
+
+ final Task task = getTask(executionAttemptID);
+
+ if (task != null) {
+ task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp);
+
+ return Acknowledge.get();
+ } else {
+ final String message = "TaskManager received a checkpoint request for unknown task " + executionAttemptID + '.';
+
+ log.debug(message);
+ throw new CheckpointException(message);
+ }
+ }
+
+ @RpcMethod
+ public Acknowledge confirmCheckpoint(ExecutionAttemptID executionAttemptID, long checkpointId, long checkpointTimestamp) throws CheckpointException {
+ log.debug("Confirm checkpoint {}@{} for {}.", checkpointId, checkpointTimestamp, executionAttemptID);
+
+ final Task task = getTask(executionAttemptID);
+
+ if (task != null) {
+ task.notifyCheckpointComplete(checkpointId);
+
+ return Acknowledge.get();
+ } else {
+ final String message = "TaskManager received a checkpoint confirmation for unknown task " + executionAttemptID + '.';
+
+ log.debug(message);
+ throw new CheckpointException(message);
+ }
+ }
+
+ /**
+ * Requests a slot from the TaskManager
+ *
+ * @param slotID Slot id for the request
+ * @param allocationID id for the request
+ * @param resourceManagerLeaderID current leader id of the ResourceManager
+ * @return answer to the slot request
+ */
+ @RpcMethod
+ public TMSlotRequestReply requestSlot(SlotID slotID, AllocationID allocationID, UUID resourceManagerLeaderID) {
+ if (!resourceManagerConnection.getTargetLeaderId().equals(resourceManagerLeaderID)) {
+ return new TMSlotRequestRejected(
+ resourceManagerConnection.getRegistrationId(), getResourceID(), allocationID);
+ }
+ if (unconfirmedFreeSlots.contains(slotID)) {
+ // check if request has not been blacklisted because the notification of a free slot
+ // has not been confirmed by the ResourceManager
+ return new TMSlotRequestRejected(
+ resourceManagerConnection.getRegistrationId(), getResourceID(), allocationID);
+ }
+ return new TMSlotRequestRegistered(new InstanceID(), ResourceID.generate(), allocationID);
+
+ }
+
// ------------------------------------------------------------------------
- // RPC methods - ResourceManager related
+ // Internal methods
// ------------------------------------------------------------------------
- @RpcMethod
- public void notifyOfNewResourceManagerLeader(String newLeaderAddress, UUID newLeaderId) {
+ private JobManagerConnection getJobManagerConnection(ResourceID jobManagerID) {
+ return jobManagerConnections.get(jobManagerID);
+ }
+
+ private Task getTask(ExecutionAttemptID executionAttemptID) {
+ TaskSlotMapping taskSlotMapping = taskSlotMappings.get(executionAttemptID);
+
+ if (taskSlotMapping != null) {
+ return taskSlotMapping.getTask();
+ } else {
+ return null;
+ }
+ }
+
+ private Task removeTask(ExecutionAttemptID executionAttemptID) {
+ TaskSlotMapping taskSlotMapping = taskSlotMappings.remove(executionAttemptID);
+
+ if (taskSlotMapping != null) {
+ final Task task = taskSlotMapping.getTask();
+ final TaskSlot taskSlot = taskSlotMapping.getTaskSlot();
+
+ taskSlot.remove(task);
+
+ return task;
+ } else {
+ return null;
+ }
+ }
+
+ private Iterable<Task> getAllTasks() {
+ final Iterator<TaskSlotMapping> taskEntryIterator = taskSlotMappings.values().iterator();
+ final Iterator<Task> iterator = new Iterator<Task>() {
+ @Override
+ public boolean hasNext() {
+ return taskEntryIterator.hasNext();
+ }
+
+ @Override
+ public Task next() {
+ return taskEntryIterator.next().getTask();
+ }
+
+ @Override
+ public void remove() {
+ taskEntryIterator.remove();
+ }
+ };
+
+ return new Iterable<Task>() {
+ @Override
+ public Iterator<Task> iterator() {
+ return iterator;
+ }
+ };
+ }
+
+ private void clearTasks() {
+ taskSlotMappings.clear();
+
+ for (TaskSlot taskSlot: taskSlots.values()) {
+ taskSlot.clear();
+ }
+ }
+
+ private void failTask(final ExecutionAttemptID executionAttemptID, final Throwable cause) {
+ final Task task = getTask(executionAttemptID);
+
+ if (task != null) {
+ try {
+ task.failExternally(cause);
+ } catch (Throwable t) {
+ log.error("Could not fail task {}.", executionAttemptID, t);
+ }
+ } else {
+ log.debug("Cannot find task to fail for execution {}.", executionAttemptID);
+ }
+ }
+
+ private void cancelAndClearAllTasks(Throwable cause) {
+ log.info("Cancellaing all computations and discarding all cached data.");
+
+ Iterable<Task> tasks = getAllTasks();
+
+ for (Task task: tasks) {
+ task.failExternally(cause);
+ }
+
+ clearTasks();
+ }
+
+ private void updateTaskExecutionState(final JobMasterGateway jobMasterGateway, final TaskExecutionState taskExecutionState) {
+ final ExecutionAttemptID executionAttemptID = taskExecutionState.getID();
+
+ Future<Acknowledge> futureAcknowledge = jobMasterGateway.updateTaskExecutionState(taskExecutionState);
+
+ futureAcknowledge.exceptionallyAsync(new ApplyFunction<Throwable, Void>() {
+ @Override
+ public Void apply(Throwable value) {
+ failTask(executionAttemptID, value);
+
+ return null;
+ }
+ }, getMainThreadExecutor());
+ }
+
+ private void unregisterTaskAndNotifyFinalState(final JobMasterGateway jobMasterGateway, ExecutionAttemptID executionAttemptID) {
+ Task task = removeTask(executionAttemptID);
+
+ if (task != null) {
+ if (!task.getExecutionState().isTerminal()) {
+ try {
+ task.failExternally(new IllegalStateException("Task is being remove from TaskManager."));
+ } catch (Exception e) {
+ log.error("Could not properly fail task.", e);
+ }
+ }
+
+ log.info("Un-registering task and sending final execution state {} to JobManager for task {} {}.",
+ task.getExecutionState(), task.getTaskInfo().getTaskName(), task.getExecutionId());
+
+ AccumulatorSnapshot accumulatorSnapshot = task.getAccumulatorRegistry().getSnapshot();
+
+ updateTaskExecutionState(
+ jobMasterGateway,
+ new TaskExecutionState(
+ task.getJobID(),
+ task.getExecutionId(),
+ task.getExecutionState(),
+ task.getFailureCause(),
+ accumulatorSnapshot));
+ } else {
+ log.error("Cannot find task with ID {} to unregister.", executionAttemptID);
+ }
+ }
+
+ private void notifyOfNewResourceManagerLeader(String newLeaderAddress, UUID newLeaderId) {
if (resourceManagerConnection != null) {
if (newLeaderAddress != null) {
// the resource manager switched to a new leader
@@ -178,28 +670,46 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
- /**
- * Requests a slot from the TaskManager
- *
- * @param slotID Slot id for the request
- * @param allocationID id for the request
- * @param resourceManagerLeaderID current leader id of the ResourceManager
- * @return answer to the slot request
- */
- @RpcMethod
- public TMSlotRequestReply requestSlot(SlotID slotID, AllocationID allocationID, UUID resourceManagerLeaderID) {
- if (!resourceManagerConnection.getTargetLeaderId().equals(resourceManagerLeaderID)) {
- return new TMSlotRequestRejected(
- resourceManagerConnection.getRegistrationId(), getResourceID(), allocationID);
- }
- if (unconfirmedFreeSlots.contains(slotID)) {
- // check if request has not been blacklisted because the notification of a free slot
- // has not been confirmed by the ResourceManager
- return new TMSlotRequestRejected(
- resourceManagerConnection.getRegistrationId(), getResourceID(), allocationID);
- }
- return new TMSlotRequestRegistered(new InstanceID(), ResourceID.generate(), allocationID);
+ private JobManagerConnection associateWithJobManager(JobMasterGateway jobMasterGateway, int blobPort) {
+ Preconditions.checkNotNull(jobMasterGateway);
+ Preconditions.checkArgument(blobPort > 0 || blobPort <= 65535, "Blob port is out of range.");
+ TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobMasterGateway);
+
+ CheckpointResponder checkpointResponder = new RpcCheckpointResponder(jobMasterGateway);
+
+ InetSocketAddress address = new InetSocketAddress(jobMasterGateway.getAddress(), blobPort);
+
+ BlobCache blobCache = new BlobCache(address, taskManagerConfiguration.getConfiguration());
+
+ LibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(
+ blobCache,
+ taskManagerConfiguration.getCleanupInterval());
+
+ ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = new RpcResultPartitionConsumableNotifier(
+ jobMasterGateway,
+ getRpcService().getExecutor(),
+ taskManagerConfiguration.getTimeout());
+
+ PartitionStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobMasterGateway);
+
+ return new JobManagerConnection(
+ jobMasterGateway,
+ taskManagerActions,
+ checkpointResponder,
+ libraryCacheManager,
+ resultPartitionConsumableNotifier,
+ partitionStateChecker);
+ }
+
+ private void disassociateFromJobManager(JobManagerConnection jobManagerConnection) throws IOException {
+ if (jobManagerConnection != null) {
+ JobMasterGateway jobManagerGateway = jobManagerConnection.getJobManagerGateway();
+
+ jobManagerGateway.disconnectTaskManager(getResourceID());
+
+ jobManagerConnection.getLibraryCacheManager().shutdown();
+ }
}
// ------------------------------------------------------------------------
@@ -237,8 +747,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
* @param t The exception describing the fatal error
*/
void onFatalError(Throwable t) {
- // to be determined, probably delegate to a fatal error handler that
- // would either log (mini cluster) ot kill the process (yarn, mesos, ...)
+ log.error("Fatal error occurred.", t);
fatalErrorHandler.onFatalError(t);
}
@@ -266,8 +775,13 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
private class ResourceManagerLeaderListener implements LeaderRetrievalListener {
@Override
- public void notifyLeaderAddress(String leaderAddress, UUID leaderSessionID) {
- getSelf().notifyOfNewResourceManagerLeader(leaderAddress, leaderSessionID);
+ public void notifyLeaderAddress(final String leaderAddress, final UUID leaderSessionID) {
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ notifyOfNewResourceManagerLeader(leaderAddress, leaderSessionID);
+ }
+ });
}
@Override
@@ -276,4 +790,43 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
+ private class TaskManagerActionsImpl implements TaskManagerActions {
+ private final JobMasterGateway jobMasterGateway;
+
+ private TaskManagerActionsImpl(JobMasterGateway jobMasterGateway) {
+ this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
+ }
+
+ @Override
+ public void notifyFinalState(final ExecutionAttemptID executionAttemptID) {
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ unregisterTaskAndNotifyFinalState(jobMasterGateway, executionAttemptID);
+ }
+ });
+ }
+
+ @Override
+ public void notifyFatalError(String message, Throwable cause) {
+ log.error(message, cause);
+ fatalErrorHandler.onFatalError(cause);
+ }
+
+ @Override
+ public void failTask(final ExecutionAttemptID executionAttemptID, final Throwable cause) {
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ TaskExecutor.this.failTask(executionAttemptID, cause);
+ }
+ });
+ }
+
+ @Override
+ public void updateTaskExecutionState(final TaskExecutionState taskExecutionState) {
+ TaskExecutor.this.updateTaskExecutionState(jobMasterGateway, taskExecutionState);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
index 2360b53..f062b96 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
@@ -21,11 +21,18 @@ package org.apache.flink.runtime.taskexecutor;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.SlotID;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
+import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.executiongraph.PartitionInfo;
+import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.rpc.RpcGateway;
import org.apache.flink.runtime.rpc.RpcTimeout;
+import org.apache.flink.runtime.taskmanager.Task;
+import java.util.Collection;
import java.util.UUID;
/**
@@ -33,12 +40,6 @@ import java.util.UUID;
*/
public interface TaskExecutorGateway extends RpcGateway {
- // ------------------------------------------------------------------------
- // ResourceManager handlers
- // ------------------------------------------------------------------------
-
- void notifyOfNewResourceManagerLeader(String address, UUID resourceManagerLeaderId);
-
/**
* Requests a slot from the TaskManager
*
@@ -52,4 +53,71 @@ public interface TaskExecutorGateway extends RpcGateway {
AllocationID allocationID,
UUID resourceManagerLeaderID,
@RpcTimeout Time timeout);
+
+ /**
+ * Submit a {@link Task} to the {@link TaskExecutor}.
+ *
+ * @param tdd describing the task to submit
+ * @param jobManagerID identifying the submitting JobManager
+ * @param timeout of the submit operation
+ * @return Future acknowledge of the successful operation
+ */
+ Future<Acknowledge> submitTask(
+ TaskDeploymentDescriptor tdd,
+ ResourceID jobManagerID,
+ @RpcTimeout Time timeout);
+
+ /**
+ * Update the task where the given partitions can be found.
+ *
+ * @param executionAttemptID identifying the task
+ * @param partitionInfos telling where the partition can be retrieved from
+ * @return Future acknowledge if the partitions have been successfully updated
+ */
+ Future<Acknowledge> updatePartitions(ExecutionAttemptID executionAttemptID, Collection<PartitionInfo> partitionInfos);
+
+ /**
+ * Fail all intermediate result partitions of the given task.
+ *
+ * @param executionAttemptID identifying the task
+ */
+ void failPartition(ExecutionAttemptID executionAttemptID);
+
+ /**
+ * Trigger the checkpoint for the given task. The checkpoint is identified by the checkpoint ID
+ * and the checkpoint timestamp.
+ *
+ * @param executionAttemptID identifying the task
+ * @param checkpointID unique id for the checkpoint
+ * @param checkpointTimestamp is the timestamp when the checkpoint has been initiated
+ * @return Future acknowledge if the checkpoint has been successfully triggered
+ */
+ Future<Acknowledge> triggerCheckpoint(ExecutionAttemptID executionAttemptID, long checkpointID, long checkpointTimestamp);
+
+ /**
+ * Confirm a checkpoint for the given task. The checkpoint is identified by the checkpoint ID
+ * and the checkpoint timestamp.
+ *
+ * @param executionAttemptID identifying the task
+ * @param checkpointId unique id for the checkpoint
+ * @param checkpointTimestamp is the timestamp when the checkpoint has been initiated
+ * @return Future acknowledge if the checkpoint has been successfully confirmed
+ */
+ Future<Acknowledge> confirmCheckpoint(ExecutionAttemptID executionAttemptID, long checkpointId, long checkpointTimestamp);
+
+ /**
+ * Stop the given task.
+ *
+ * @param executionAttemptID identifying the task
+ * @return Future acknowledge if the task is successfully stopped
+ */
+ Future<Acknowledge> stopTask(ExecutionAttemptID executionAttemptID);
+
+ /**
+ * Cancel the given task.
+ *
+ * @param executionAttemptID identifying the task
+ * @return Future acknowledge if the task is successfully canceled
+ */
+ Future<Acknowledge> cancelTask(ExecutionAttemptID executionAttemptID);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
index f58af77..bce3dc3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.taskexecutor;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.UnmodifiableConfiguration;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
@@ -71,7 +72,7 @@ public class TaskManagerConfiguration {
this.maxRegistrationPause = Preconditions.checkNotNull(maxRegistrationPause);
this.refusedRegistrationPause = Preconditions.checkNotNull(refusedRegistrationPause);
this.cleanupInterval = Preconditions.checkNotNull(cleanupInterval);
- this.configuration = Preconditions.checkNotNull(configuration);
+ this.configuration = new UnmodifiableConfiguration(Preconditions.checkNotNull(configuration));
}
public int getNumberSlots() {
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
index 8ac0ddd..bb66655 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
@@ -95,6 +95,9 @@ public class TaskManagerRunner implements FatalErrorHandler {
taskManagerServices.getNetworkEnvironment(),
highAvailabilityServices,
taskManagerServices.getMetricRegistry(),
+ taskManagerServices.getTaskManagerMetricGroup(),
+ taskManagerServices.getBroadcastVariableManager(),
+ taskManagerServices.getFileCache(),
this);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
index ff7f7d5..e264a1c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
@@ -19,7 +19,9 @@
package org.apache.flink.runtime.taskexecutor;
import org.apache.flink.core.memory.MemoryType;
+import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.filecache.FileCache;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync;
import org.apache.flink.runtime.io.network.ConnectionManager;
@@ -32,9 +34,11 @@ import org.apache.flink.runtime.io.network.netty.NettyConnectionManager;
import org.apache.flink.runtime.io.network.partition.ResultPartitionManager;
import org.apache.flink.runtime.memory.MemoryManager;
import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
import org.apache.flink.runtime.query.KvStateRegistry;
import org.apache.flink.runtime.query.netty.DisabledKvStateRequestStats;
import org.apache.flink.runtime.query.netty.KvStateServer;
+import org.apache.flink.runtime.taskexecutor.utils.TaskExecutorMetricsInitializer;
import org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.runtime.util.EnvironmentInformation;
@@ -58,19 +62,28 @@ public class TaskManagerServices {
private final IOManager ioManager;
private final NetworkEnvironment networkEnvironment;
private final MetricRegistry metricRegistry;
+ private final TaskManagerMetricGroup taskManagerMetricGroup;
+ private final BroadcastVariableManager broadcastVariableManager;
+ private final FileCache fileCache;
private TaskManagerServices(
TaskManagerLocation taskManagerLocation,
MemoryManager memoryManager,
IOManager ioManager,
NetworkEnvironment networkEnvironment,
- MetricRegistry metricRegistry) {
+ MetricRegistry metricRegistry,
+ TaskManagerMetricGroup taskManagerMetricGroup,
+ BroadcastVariableManager broadcastVariableManager,
+ FileCache fileCache) {
this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
this.memoryManager = Preconditions.checkNotNull(memoryManager);
this.ioManager = Preconditions.checkNotNull(ioManager);
this.networkEnvironment = Preconditions.checkNotNull(networkEnvironment);
this.metricRegistry = Preconditions.checkNotNull(metricRegistry);
+ this.taskManagerMetricGroup = Preconditions.checkNotNull(taskManagerMetricGroup);
+ this.broadcastVariableManager = Preconditions.checkNotNull(broadcastVariableManager);
+ this.fileCache = Preconditions.checkNotNull(fileCache);
}
// --------------------------------------------------------------------------------------------
@@ -97,6 +110,18 @@ public class TaskManagerServices {
return metricRegistry;
}
+ public TaskManagerMetricGroup getTaskManagerMetricGroup() {
+ return taskManagerMetricGroup;
+ }
+
+ public BroadcastVariableManager getBroadcastVariableManager() {
+ return broadcastVariableManager;
+ }
+
+ public FileCache getFileCache() {
+ return fileCache;
+ }
+
// --------------------------------------------------------------------------------------------
// Static factory methods for task manager services
// --------------------------------------------------------------------------------------------
@@ -128,9 +153,29 @@ public class TaskManagerServices {
// start the I/O manager, it will create some temp directories.
final IOManager ioManager = new IOManagerAsync(taskManagerServicesConfiguration.getTmpDirPaths());
- MetricRegistry metricsRegistry = new MetricRegistry(taskManagerServicesConfiguration.getMetricRegistryConfiguration());
+ final MetricRegistry metricRegistry = new MetricRegistry(taskManagerServicesConfiguration.getMetricRegistryConfiguration());
+
+ final TaskManagerMetricGroup taskManagerMetricGroup = new TaskManagerMetricGroup(
+ metricRegistry,
+ taskManagerLocation.getHostname(),
+ taskManagerLocation.getResourceID().toString());
+
+ // Initialize the TM metrics
+ TaskExecutorMetricsInitializer.instantiateStatusMetrics(taskManagerMetricGroup, network);
+
+ final BroadcastVariableManager broadcastVariableManager = new BroadcastVariableManager();
+
+ final FileCache fileCache = new FileCache(taskManagerServicesConfiguration.getTmpDirPaths());
- return new TaskManagerServices(taskManagerLocation, memoryManager, ioManager, network, metricsRegistry);
+ return new TaskManagerServices(
+ taskManagerLocation,
+ memoryManager,
+ ioManager,
+ network,
+ metricRegistry,
+ taskManagerMetricGroup,
+ broadcastVariableManager,
+ fileCache);
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlot.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlot.java
new file mode 100644
index 0000000..4fc1d66
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlot.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor;
+
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.taskmanager.Task;
+import org.apache.flink.util.Preconditions;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Container for multiple {@link Task} belonging to the same slot.
+ */
+public class TaskSlot {
+ private final AllocationID allocationID;
+ private final ResourceID resourceID;
+ private final Map<ExecutionAttemptID, Task> tasks;
+
+ public TaskSlot(AllocationID allocationID, ResourceID resourceID) {
+ this.allocationID = Preconditions.checkNotNull(allocationID);
+ this.resourceID = Preconditions.checkNotNull(resourceID);
+ tasks = new HashMap<>(4);
+ }
+
+ public AllocationID getAllocationID() {
+ return allocationID;
+ }
+
+ public ResourceID getResourceID() {
+ return resourceID;
+ }
+
+ public boolean add(Task task) {
+ // sanity check
+ Preconditions.checkArgument(allocationID.equals(task.getAllocationID()));
+
+ Task oldTask = tasks.put(task.getExecutionId(), task);
+
+ if (oldTask != null) {
+ tasks.put(task.getExecutionId(), oldTask);
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ public Task remove(Task task) {
+ return tasks.remove(task.getExecutionId());
+ }
+
+ public void clear() {
+ tasks.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlotMapping.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlotMapping.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlotMapping.java
new file mode 100644
index 0000000..e67fd52
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlotMapping.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor;
+
+import org.apache.flink.runtime.taskmanager.Task;
+import org.apache.flink.util.Preconditions;
+
+/**
+ * Mapping between a {@link Task} and its {@link TaskSlot}.
+ */
+public class TaskSlotMapping {
+
+ private final Task task;
+ private final TaskSlot taskSlot;
+
+ public TaskSlotMapping(Task task, TaskSlot taskSlot) {
+ this.task = Preconditions.checkNotNull(task);
+ this.taskSlot = Preconditions.checkNotNull(taskSlot);
+ }
+
+ public Task getTask() {
+ return task;
+ }
+
+ public TaskSlot getTaskSlot() {
+ return taskSlot;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0655adca/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/CheckpointException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/CheckpointException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/CheckpointException.java
new file mode 100644
index 0000000..80f2aa0
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/CheckpointException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.exceptions;
+
+import org.apache.flink.runtime.taskexecutor.TaskExecutor;
+
+/**
+ * Exception indicating a problem with checkpointing on the {@link TaskExecutor} side.
+ */
+public class CheckpointException extends TaskManagerException {
+
+ private static final long serialVersionUID = 3366394086880327955L;
+
+ public CheckpointException(String message) {
+ super(message);
+ }
+
+ public CheckpointException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public CheckpointException(Throwable cause) {
+ super(cause);
+ }
+}
[49/50] [abbrv] flink git commit: [FLINK-4853] [rm] Clean up job
manager registration at the resource manager
Posted by tr...@apache.org.
[FLINK-4853] [rm] Clean up job manager registration at the resource manager
Introduce the JobLeaderIdService which automatically retrieves the current job leader id.
This job leader id is used to validate job manager registartion attempts. Additionally, it
is used to disconnect old job leaders from the resource manager.
Add comments
This closes #2657.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2a5a5b39
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2a5a5b39
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2a5a5b39
Branch: refs/heads/flip-6
Commit: 2a5a5b3952c407bf3c09b461a2267b6a369d7960
Parents: 7fab3ed
Author: Till Rohrmann <tr...@apache.org>
Authored: Tue Oct 18 18:03:00 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:35 2016 +0100
----------------------------------------------------------------------
.../flink/runtime/jobmaster/JobMaster.java | 10 +-
.../runtime/jobmaster/JobMasterGateway.java | 12 +
.../resourcemanager/JobLeaderIdActions.java | 51 ++
.../resourcemanager/JobLeaderIdService.java | 285 ++++++++++
.../resourcemanager/ResourceManager.java | 538 ++++++++++---------
.../resourcemanager/ResourceManagerGateway.java | 2 +-
.../StandaloneResourceManager.java | 5 +-
.../exceptions/ResourceManagerRunner.java | 5 +-
.../registration/JobManagerRegistration.java | 58 ++
.../registration/JobMasterRegistration.java | 62 ---
.../flink/runtime/rpc/akka/AkkaRpcService.java | 2 -
.../resourcemanager/ResourceManagerHATest.java | 8 +-
.../ResourceManagerJobMasterTest.java | 14 +-
.../ResourceManagerTaskExecutorTest.java | 2 +
.../slotmanager/SlotProtocolTest.java | 12 +-
15 files changed, 736 insertions(+), 330 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 49c4df1..8df3a3e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -758,6 +758,14 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
}
}
+ @RpcMethod
+ public void disconnectResourceManager(
+ final UUID jobManagerLeaderId,
+ final UUID resourceManagerLeaderId,
+ final Exception cause) {
+ // TODO: Implement disconnect behaviour
+ }
+
//----------------------------------------------------------------------------------------------
// Internal methods
//----------------------------------------------------------------------------------------------
@@ -948,7 +956,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
long timeoutMillis) throws Exception
{
Time timeout = Time.milliseconds(timeoutMillis);
- return gateway.registerJobMaster(leaderId, jobManagerLeaderID, getAddress(), jobID, timeout);
+ return gateway.registerJobManager(leaderId, jobManagerLeaderID, getAddress(), jobID, timeout);
}
};
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
index 27308d3..a47623b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
@@ -130,6 +130,18 @@ public interface JobMasterGateway extends CheckpointCoordinatorGateway {
void disconnectTaskManager(ResourceID resourceID);
/**
+ * Disconnects the resource manager from the job manager because of the given cause.
+ *
+ * @param jobManagerLeaderId identifying the job manager leader id
+ * @param resourceManagerLeaderId identifying the resource manager leader id
+ * @param cause of the disconnect
+ */
+ void disconnectResourceManager(
+ final UUID jobManagerLeaderId,
+ final UUID resourceManagerLeaderId,
+ final Exception cause);
+
+ /**
* Requests a {@link KvStateLocation} for the specified {@link KvState} registration name.
*
* @param registrationName Name under which the KvState has been registered.
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdActions.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdActions.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdActions.java
new file mode 100644
index 0000000..58777ef
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdActions.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager;
+
+import org.apache.flink.api.common.JobID;
+
+import java.util.UUID;
+
+/**
+ * Interface for actions called by the {@link JobLeaderIdService}.
+ */
+public interface JobLeaderIdActions {
+
+ /**
+ * Callback when a monitored job leader lost its leadership.
+ *
+ * @param jobId identifying the job whose leader lost leadership
+ * @param oldJobLeaderId of the job manager which lost leadership
+ */
+ void jobLeaderLostLeadership(JobID jobId, UUID oldJobLeaderId);
+
+ /**
+ * Request to remove the job from the {@link JobLeaderIdService}.
+ *
+ * @param jobId identifying the job to remove
+ */
+ void removeJob(JobID jobId);
+
+ /**
+ * Callback to report occurring errors.
+ *
+ * @param error which has occurred
+ */
+ void handleError(Throwable error);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdService.java
new file mode 100644
index 0000000..6c7e249
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdService.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.concurrent.CompletableFuture;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.RunningJobsRegistry;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Service which retrieves for a registered job the current job leader id (the leader id of the
+ * job manager responsible for the job). The leader id will be exposed as a future via the
+ * {@link #getLeaderId(JobID)}. The future will only be completed with an exception in case
+ * the service will be stopped.
+ */
+public class JobLeaderIdService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(JobLeaderIdService.class);
+
+ /** High availability services to use by this service */
+ private final HighAvailabilityServices highAvailabilityServices;
+
+ /** Registry to retrieve running jobs */
+ private final RunningJobsRegistry runningJobsRegistry;
+
+ /** Map of currently monitored jobs */
+ private final Map<JobID, JobLeaderIdListener> jobLeaderIdListeners;
+
+ /** Actions to call when the job leader changes */
+ private JobLeaderIdActions jobLeaderIdActions;
+
+ public JobLeaderIdService(HighAvailabilityServices highAvailabilityServices) throws Exception {
+ this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices);
+
+ this.runningJobsRegistry = highAvailabilityServices.getRunningJobsRegistry();
+
+ jobLeaderIdListeners = new HashMap<>(4);
+
+ jobLeaderIdActions = null;
+ }
+
+ /**
+ * Start the service with the given job leader actions.
+ *
+ * @param initialJobLeaderIdActions to use for job leader id actions
+ * @throws Exception which is thrown when clearing up old state
+ */
+ public void start(JobLeaderIdActions initialJobLeaderIdActions) throws Exception {
+ if (isStarted()) {
+ clear();
+ }
+
+ this.jobLeaderIdActions = Preconditions.checkNotNull(initialJobLeaderIdActions);
+ }
+
+ /**
+ * Stop the service.
+ *
+ * @throws Exception which is thrown in case a retrieval service cannot be stopped properly
+ */
+ public void stop() throws Exception {
+ clear();
+
+ this.jobLeaderIdActions = null;
+ }
+
+ /**
+ * Checks whether the service has been started.
+ *
+ * @return True if the service has been started; otherwise false
+ */
+ public boolean isStarted() {
+ return jobLeaderIdActions == null;
+ }
+
+ /**
+ * Stop and clear the currently registered job leader id listeners.
+ *
+ * @throws Exception which is thrown in case a retrieval service cannot be stopped properly
+ */
+ public void clear() throws Exception {
+ Exception exception = null;
+
+ for (JobLeaderIdListener listener: jobLeaderIdListeners.values()) {
+ try {
+ listener.stop();
+ } catch (Exception e) {
+ exception = ExceptionUtils.firstOrSuppressed(e, exception);
+ }
+ }
+
+ if (exception != null) {
+ ExceptionUtils.rethrowException(exception, "Could not properly stop the " +
+ JobLeaderIdService.class.getSimpleName() + '.');
+ }
+
+ jobLeaderIdListeners.clear();
+ }
+
+ /**
+ * Add a job to be monitored to retrieve the job leader id.
+ *
+ * @param jobId identifying the job to monitor
+ * @throws Exception if the job could not be added to the service
+ */
+ public void addJob(JobID jobId) throws Exception {
+ Preconditions.checkNotNull(jobLeaderIdActions);
+
+ LOG.debug("Add job {} to job leader id monitoring.", jobId);
+
+ if (!jobLeaderIdListeners.containsKey(jobId)) {
+ LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(jobId);
+
+ JobLeaderIdListener jobidListener = new JobLeaderIdListener(jobId, jobLeaderIdActions, leaderRetrievalService);
+ jobLeaderIdListeners.put(jobId, jobidListener);
+ }
+ }
+
+ /**
+ * Remove the given job from being monitored by the service.
+ *
+ * @param jobId identifying the job to remove from monitor
+ * @throws Exception if removing the job fails
+ */
+ public void removeJob(JobID jobId) throws Exception {
+ LOG.debug("Remove job {} from job leader id monitoring.", jobId);
+
+ JobLeaderIdListener listener = jobLeaderIdListeners.remove(jobId);
+
+ if (listener != null) {
+ listener.stop();
+ }
+ }
+
+ /**
+ * Check whether the given job is being monitored or not.
+ *
+ * @param jobId identifying the job
+ * @return True if the job is being monitored; otherwise false
+ */
+ public boolean containsJob(JobID jobId) {
+ return jobLeaderIdListeners.containsKey(jobId);
+ }
+
+ public Future<UUID> getLeaderId(JobID jobId) throws Exception {
+ if (!jobLeaderIdListeners.containsKey(jobId)) {
+ addJob(jobId);
+ }
+
+ JobLeaderIdListener listener = jobLeaderIdListeners.get(jobId);
+
+ return listener.getLeaderIdFuture();
+ }
+
+ // --------------------------------------------------------------------------------
+ // Static utility classes
+ // --------------------------------------------------------------------------------
+
+ /**
+ * Listener which stores the current leader id and exposes them as a future value when
+ * requested. The returned future will always be completed properly except when stopping the
+ * listener.
+ */
+ private final class JobLeaderIdListener implements LeaderRetrievalListener {
+ private final JobID jobId;
+ private final JobLeaderIdActions listenerJobLeaderIdActions;
+ private final LeaderRetrievalService leaderRetrievalService;
+
+ private volatile CompletableFuture<UUID> leaderIdFuture;
+ private volatile boolean running = true;
+
+ private JobLeaderIdListener(
+ JobID jobId,
+ JobLeaderIdActions listenerJobLeaderIdActions,
+ LeaderRetrievalService leaderRetrievalService) throws Exception {
+ this.jobId = Preconditions.checkNotNull(jobId);
+ this.listenerJobLeaderIdActions = Preconditions.checkNotNull(listenerJobLeaderIdActions);
+ this.leaderRetrievalService = Preconditions.checkNotNull(leaderRetrievalService);
+
+ leaderIdFuture = new FlinkCompletableFuture<>();
+
+ // start the leader service we're listening to
+ leaderRetrievalService.start(this);
+ }
+
+ public Future<UUID> getLeaderIdFuture() {
+ return leaderIdFuture;
+ }
+
+ public void stop() throws Exception {
+ running = false;
+ leaderRetrievalService.stop();
+ leaderIdFuture.completeExceptionally(new Exception("Job leader id service has been stopped."));
+ }
+
+ @Override
+ public void notifyLeaderAddress(String leaderAddress, UUID leaderSessionId) {
+ if (running) {
+ LOG.debug("Found a new job leader {}@{}.", leaderSessionId, leaderAddress);
+
+ UUID previousJobLeaderId = null;
+
+ if (leaderIdFuture.isDone()) {
+ try {
+ previousJobLeaderId = leaderIdFuture.getNow(null);
+ } catch (ExecutionException e) {
+ // this should never happen since we complete this future always properly
+ handleError(e);
+ }
+
+ leaderIdFuture = FlinkCompletableFuture.completed(leaderSessionId);
+ } else {
+ leaderIdFuture.complete(leaderSessionId);
+ }
+
+ try {
+ if (runningJobsRegistry.isJobRunning(jobId)) {
+ if (leaderSessionId == null) {
+ // there is no new leader
+ if (previousJobLeaderId != null) {
+ // we had a previous job leader, so notify about his lost leadership
+ listenerJobLeaderIdActions.jobLeaderLostLeadership(jobId, previousJobLeaderId);
+ }
+ } else {
+ if (previousJobLeaderId != null && !leaderSessionId.equals(previousJobLeaderId)) {
+ // we had a previous leader and he's not the same as the new leader
+ listenerJobLeaderIdActions.jobLeaderLostLeadership(jobId, previousJobLeaderId);
+ }
+ }
+ } else {
+ // the job is no longer running so remove it
+ listenerJobLeaderIdActions.removeJob(jobId);
+ }
+ } catch (IOException e) {
+ // cannot tell whether the job is still running or not so just remove the listener
+ LOG.debug("Encountered an error while checking the job registry for running jobs.", e);
+ listenerJobLeaderIdActions.removeJob(jobId);
+ }
+ } else {
+ LOG.debug("A leader id change {}@{} has been detected after the listener has been stopped.",
+ leaderSessionId, leaderAddress);
+ }
+ }
+
+ @Override
+ public void handleError(Exception exception) {
+ if (running) {
+ listenerJobLeaderIdActions.handleError(exception);
+ } else {
+ LOG.debug("An error occurred in the {} after the listener has been stopped.",
+ JobLeaderIdListener.class.getSimpleName(), exception);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index 4161972..7240087 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -20,7 +20,6 @@ package org.apache.flink.runtime.resourcemanager;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.clusterframework.ApplicationStatus;
import org.apache.flink.runtime.clusterframework.messages.InfoMessage;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
@@ -36,13 +35,11 @@ import org.apache.flink.runtime.instance.InstanceID;
import org.apache.flink.runtime.jobmaster.JobMasterRegistrationSuccess;
import org.apache.flink.runtime.leaderelection.LeaderContender;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestRejected;
import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestReply;
-import org.apache.flink.runtime.resourcemanager.registration.JobMasterRegistration;
+import org.apache.flink.runtime.resourcemanager.registration.JobManagerRegistration;
import org.apache.flink.runtime.resourcemanager.registration.WorkerRegistration;
import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
@@ -61,10 +58,8 @@ import org.apache.flink.util.ExceptionUtils;
import java.io.Serializable;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.Map;
import java.util.UUID;
-import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executor;
@@ -77,7 +72,7 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
*
* It offers the following methods as part of its rpc interface to interact with him remotely:
* <ul>
- * <li>{@link #registerJobMaster(UUID, UUID, String, JobID)} registers a {@link JobMaster} at the resource manager</li>
+ * <li>{@link #registerJobManager(UUID, UUID, String, JobID)} registers a {@link JobMaster} at the resource manager</li>
* <li>{@link #requestSlot(UUID, UUID, SlotRequest)} requests a slot from the resource manager</li>
* </ul>
*/
@@ -89,10 +84,10 @@ public abstract class ResourceManager<WorkerType extends Serializable>
private final ResourceManagerConfiguration resourceManagerConfiguration;
/** All currently registered JobMasterGateways scoped by JobID. */
- private final Map<JobID, JobMasterRegistration> jobMasters;
+ private final Map<JobID, JobManagerRegistration> jobManagerRegistrations;
- /** LeaderListeners for all registered JobIDs. */
- private final Map<JobID, JobIdLeaderListener> leaderListeners;
+ /** Service to retrieve the job leader ids */
+ private final JobLeaderIdService jobLeaderIdService;
/** All currently registered TaskExecutors with there framework specific worker information. */
private final Map<ResourceID, WorkerRegistration<WorkerType>> taskExecutors;
@@ -116,7 +111,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
private LeaderElectionService leaderElectionService;
/** ResourceManager's leader session id which is updated on leader election. */
- private volatile UUID leaderSessionID;
+ private volatile UUID leaderSessionId;
/** All registered listeners for status updates of the ResourceManager. */
private ConcurrentMap<String, InfoMessageListenerRpcGateway> infoMessageListeners;
@@ -127,6 +122,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
HighAvailabilityServices highAvailabilityServices,
SlotManagerFactory slotManagerFactory,
MetricRegistry metricRegistry,
+ JobLeaderIdService jobLeaderIdService,
FatalErrorHandler fatalErrorHandler) {
super(rpcService);
@@ -135,12 +131,12 @@ public abstract class ResourceManager<WorkerType extends Serializable>
this.highAvailabilityServices = checkNotNull(highAvailabilityServices);
this.slotManagerFactory = checkNotNull(slotManagerFactory);
this.metricRegistry = checkNotNull(metricRegistry);
+ this.jobLeaderIdService = checkNotNull(jobLeaderIdService);
this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
- this.jobMasters = new HashMap<>(4);
- this.leaderListeners = new HashMap<>(4);
+ this.jobManagerRegistrations = new HashMap<>(4);
this.taskExecutors = new HashMap<>(8);
- this.leaderSessionID = null;
+ this.leaderSessionId = null;
infoMessageListeners = new ConcurrentHashMap<>(8);
}
@@ -168,11 +164,12 @@ public abstract class ResourceManager<WorkerType extends Serializable>
}
try {
- // framework specific initialization
- initialize();
+ jobLeaderIdService.start(new JobLeaderIdActionsImpl());
} catch (Exception e) {
- throw new ResourceManagerException("Could not initialize the resource manager.", e);
+ throw new ResourceManagerException("Could not start the job leader id service.", e);
}
+
+ initialize();
}
@Override
@@ -180,6 +177,12 @@ public abstract class ResourceManager<WorkerType extends Serializable>
Exception exception = null;
try {
+ jobLeaderIdService.stop();
+ } catch (Exception e) {
+ exception = ExceptionUtils.firstOrSuppressed(e, exception);
+ }
+
+ try {
leaderElectionService.stop();
} catch (Exception e) {
exception = ExceptionUtils.firstOrSuppressed(e, exception);
@@ -202,101 +205,124 @@ public abstract class ResourceManager<WorkerType extends Serializable>
// RPC methods
// ------------------------------------------------------------------------
- /**
- * Register a {@link JobMaster} at the resource manager.
- *
- * @param resourceManagerLeaderId The fencing token for the ResourceManager leader
- * @param jobMasterAddress The address of the JobMaster that registers
- * @param jobID The Job ID of the JobMaster that registers
- * @return Future registration response
- */
@RpcMethod
- public Future<RegistrationResponse> registerJobMaster(
- final UUID resourceManagerLeaderId, final UUID jobMasterLeaderId,
- final String jobMasterAddress, final JobID jobID) {
+ public Future<RegistrationResponse> registerJobManager(
+ final UUID resourceManagerLeaderId,
+ final UUID jobManagerLeaderId,
+ final String jobManagerAddress,
+ final JobID jobId) {
+
+ checkNotNull(resourceManagerLeaderId);
+ checkNotNull(jobManagerLeaderId);
+ checkNotNull(jobManagerAddress);
+ checkNotNull(jobId);
+
+ if (isValid(resourceManagerLeaderId)) {
+ if (!jobLeaderIdService.containsJob(jobId)) {
+ try {
+ jobLeaderIdService.addJob(jobId);
+ } catch (Exception e) {
+ ResourceManagerException exception = new ResourceManagerException("Could not add the job " +
+ jobId + " to the job id leader service.", e);
+
+ onFatalErrorAsync(exception);
+
+ log.error("Could not add job {} to job leader id service.", jobId, e);
+ return FlinkCompletableFuture.completedExceptionally(exception);
+ }
+ }
- checkNotNull(jobMasterAddress);
- checkNotNull(jobID);
+ log.info("Registering job manager {}@{} for job {}.", jobManagerLeaderId, jobManagerAddress, jobId);
+
+ Future<UUID> jobLeaderIdFuture;
- // create a leader retriever in case it doesn't exist
- final JobIdLeaderListener jobIdLeaderListener;
- if (leaderListeners.containsKey(jobID)) {
- jobIdLeaderListener = leaderListeners.get(jobID);
- } else {
try {
- LeaderRetrievalService jobMasterLeaderRetriever =
- highAvailabilityServices.getJobManagerLeaderRetriever(jobID);
- jobIdLeaderListener = new JobIdLeaderListener(jobID, jobMasterLeaderRetriever);
+ jobLeaderIdFuture = jobLeaderIdService.getLeaderId(jobId);
} catch (Exception e) {
- log.warn("Failed to start JobMasterLeaderRetriever for job id {}", jobID, e);
+ // we cannot check the job leader id so let's fail
+ // TODO: Maybe it's also ok to skip this check in case that we cannot check the leader id
+ ResourceManagerException exception = new ResourceManagerException("Cannot obtain the " +
+ "job leader id future to verify the correct job leader.", e);
+
+ onFatalErrorAsync(exception);
- return FlinkCompletableFuture.<RegistrationResponse>completed(
- new RegistrationResponse.Decline("Failed to retrieve JobMasterLeaderRetriever"));
+ log.debug("Could not obtain the job leader id future to verify the correct job leader.");
+ return FlinkCompletableFuture.completedExceptionally(exception);
}
- leaderListeners.put(jobID, jobIdLeaderListener);
- }
+ Future<JobMasterGateway> jobMasterGatewayFuture = getRpcService().connect(jobManagerAddress, JobMasterGateway.class);
- return getRpcService()
- .execute(new Callable<JobMasterGateway>() {
+ Future<RegistrationResponse> registrationResponseFuture = jobMasterGatewayFuture.thenCombineAsync(jobLeaderIdFuture, new BiFunction<JobMasterGateway, UUID, RegistrationResponse>() {
@Override
- public JobMasterGateway call() throws Exception {
+ public RegistrationResponse apply(JobMasterGateway jobMasterGateway, UUID jobLeaderId) {
+ if (isValid(resourceManagerLeaderId)) {
+ if (jobLeaderId.equals(jobManagerLeaderId)) {
+ if (jobManagerRegistrations.containsKey(jobId)) {
+ JobManagerRegistration oldJobManagerRegistration = jobManagerRegistrations.get(jobId);
+
+ if (oldJobManagerRegistration.getLeaderID().equals(jobLeaderId)) {
+ // same registration
+ log.debug("Job manager {}@{} was already registered.", jobManagerLeaderId, jobManagerAddress);
+ } else {
+ // tell old job manager that he is no longer the job leader
+ disconnectJobManager(
+ oldJobManagerRegistration.getJobID(),
+ new Exception("New job leader for job " + jobId + " found."));
+
+ JobManagerRegistration jobManagerRegistration = new JobManagerRegistration(jobId, jobLeaderId, jobMasterGateway);
+ jobManagerRegistrations.put(jobId, jobManagerRegistration);
+ }
+ } else {
+ // new registration for the job
+ JobManagerRegistration jobManagerRegistration = new JobManagerRegistration(jobId, jobLeaderId, jobMasterGateway);
+
+ jobManagerRegistrations.put(jobId, jobManagerRegistration);
+ }
- if (!leaderSessionID.equals(resourceManagerLeaderId)) {
- log.warn("Discard registration from JobMaster {} at ({}) because the expected leader session ID {}" +
- " did not equal the received leader session ID {}",
- jobID, jobMasterAddress, leaderSessionID, resourceManagerLeaderId);
- throw new Exception("Invalid leader session id");
- }
+ log.info("Registered job manager {}@{} for job {}.", jobManagerLeaderId, jobManagerAddress, jobId);
- final Time timeout = resourceManagerConfiguration.getTimeout();
+ return new JobMasterRegistrationSuccess(
+ resourceManagerConfiguration.getHeartbeatInterval().toMilliseconds(),
+ getLeaderSessionId());
- if (!jobIdLeaderListener.getLeaderID().get(timeout.getSize(), timeout.getUnit())
- .equals(jobMasterLeaderId)) {
- throw new Exception("Leader Id did not match");
+ } else {
+ log.debug("The job manager leader id {} did not match the job " +
+ "leader id {}.", jobManagerLeaderId, jobLeaderId);
+ return new RegistrationResponse.Decline("Job manager leader id did not match.");
+ }
+ } else {
+ log.debug("The resource manager leader id changed {}. Discarding job " +
+ "manager registration from {}.", getLeaderSessionId(), jobManagerAddress);
+ return new RegistrationResponse.Decline("Resource manager leader id changed.");
}
-
- return getRpcService().connect(jobMasterAddress, JobMasterGateway.class)
- .get(timeout.getSize(), timeout.getUnit());
}
- })
- .handleAsync(new BiFunction<JobMasterGateway, Throwable, RegistrationResponse>() {
- @Override
- public RegistrationResponse apply(JobMasterGateway jobMasterGateway, Throwable throwable) {
-
- if (throwable != null) {
- return new RegistrationResponse.Decline(throwable.getMessage());
- } else {
- if (!leaderSessionID.equals(resourceManagerLeaderId)) {
- log.warn("Discard registration from JobMaster {} at ({}) because the expected leader session ID {}" +
- " did not equal the received leader session ID {}",
- jobID, jobMasterAddress, leaderSessionID, resourceManagerLeaderId);
- return new RegistrationResponse.Decline("Invalid leader session id");
- }
+ }, getMainThreadExecutor());
- try {
- // LeaderID should be available now, but if not we fail the registration
- UUID currentJobMasterLeaderId = jobIdLeaderListener.getLeaderID().getNow(null);
- if (currentJobMasterLeaderId == null || !currentJobMasterLeaderId.equals(jobMasterLeaderId)) {
- throw new Exception("Leader Id did not match");
+ // handle exceptions which might have occurred in one of the futures inputs of combine
+ return registrationResponseFuture.handleAsync(new BiFunction<RegistrationResponse, Throwable, RegistrationResponse>() {
+ @Override
+ public RegistrationResponse apply(RegistrationResponse registrationResponse, Throwable throwable) {
+ if (throwable != null) {
+ if (log.isDebugEnabled()) {
+ log.debug("Registration of job manager {}@{} failed.", jobManagerLeaderId, jobManagerAddress, throwable);
+ } else {
+ log.info("Registration of job manager {}@{} failed.", jobManagerLeaderId, jobManagerAddress);
}
- } catch (Exception e) {
- return new RegistrationResponse.Decline(e.getMessage());
- }
-
- final JobMasterRegistration registration =
- new JobMasterRegistration(jobID, jobMasterLeaderId, jobMasterGateway);
- final JobMasterRegistration existingRegistration = jobMasters.put(jobID, registration);
- if (existingRegistration != null) {
- log.info("Replacing JobMaster registration for newly registered JobMaster with JobID {}.", jobID);
+ return new RegistrationResponse.Decline(throwable.getMessage());
+ } else {
+ return registrationResponse;
}
- return new JobMasterRegistrationSuccess(
- resourceManagerConfiguration.getHeartbeatInterval().toMilliseconds(),
- resourceManagerLeaderId);
}
- }
- }, getMainThreadExecutor());
+ }, getRpcService().getExecutor());
+ } else {
+ log.debug("Discard register job manager message from {}, because the leader id " +
+ "{} did not match the expected leader id {}.", jobManagerAddress,
+ resourceManagerLeaderId, leaderSessionId);
+
+ return FlinkCompletableFuture.<RegistrationResponse>completed(
+ new RegistrationResponse.Decline("Resource manager leader id did not match."));
+ }
}
/**
@@ -315,7 +341,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
final ResourceID resourceID,
final SlotReport slotReport) {
- if (leaderSessionID.equals(resourceManagerLeaderId)) {
+ if (leaderSessionId.equals(resourceManagerLeaderId)) {
Future<TaskExecutorGateway> taskExecutorGatewayFuture = getRpcService().connect(taskExecutorAddress, TaskExecutorGateway.class);
return taskExecutorGatewayFuture.handleAsync(new BiFunction<TaskExecutorGateway, Throwable, RegistrationResponse>() {
@@ -346,12 +372,12 @@ public abstract class ResourceManager<WorkerType extends Serializable>
} else {
log.warn("Discard registration from TaskExecutor {} at ({}) because the expected leader session ID {} did " +
"not equal the received leader session ID {}",
- resourceID, taskExecutorAddress, leaderSessionID, resourceManagerLeaderId);
+ resourceID, taskExecutorAddress, leaderSessionId, resourceManagerLeaderId);
return FlinkCompletableFuture.<RegistrationResponse>completed(
new RegistrationResponse.Decline("Discard registration because the leader id " +
resourceManagerLeaderId + " does not match the expected leader id " +
- leaderSessionID + '.'));
+ leaderSessionId + '.'));
}
}
@@ -368,11 +394,11 @@ public abstract class ResourceManager<WorkerType extends Serializable>
SlotRequest slotRequest) {
JobID jobId = slotRequest.getJobId();
- JobMasterRegistration jobMasterRegistration = jobMasters.get(jobId);
+ JobManagerRegistration jobManagerRegistration = jobManagerRegistrations.get(jobId);
- if (jobMasterRegistration != null
- && jobMasterLeaderID.equals(jobMasterRegistration.getLeaderID())
- && resourceManagerLeaderID.equals(leaderSessionID)) {
+ if (jobManagerRegistration != null
+ && jobMasterLeaderID.equals(jobManagerRegistration.getLeaderID())
+ && resourceManagerLeaderID.equals(leaderSessionId)) {
return slotManager.requestSlot(slotRequest);
} else {
log.info("Ignoring slot request for unknown JobMaster with JobID {}", jobId);
@@ -393,7 +419,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
final InstanceID instanceID,
final SlotID slotId) {
- if (resourceManagerLeaderId.equals(leaderSessionID)) {
+ if (resourceManagerLeaderId.equals(leaderSessionId)) {
final ResourceID resourceId = slotId.getResourceID();
WorkerRegistration<WorkerType> registration = taskExecutors.get(resourceId);
@@ -413,7 +439,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
} else {
log.debug("Discarding notify slot available message for slot {}, because the " +
"leader id {} did not match the expected leader id {}.", slotId,
- resourceManagerLeaderId, leaderSessionID);
+ resourceManagerLeaderId, leaderSessionId);
}
}
@@ -471,6 +497,150 @@ public abstract class ResourceManager<WorkerType extends Serializable>
}
// ------------------------------------------------------------------------
+ // Testing methods
+ // ------------------------------------------------------------------------
+
+ /**
+ * Gets the leader session id of current resourceManager.
+ *
+ * @return return the leaderSessionId of current resourceManager, this returns null until the current resourceManager is granted leadership.
+ */
+ @VisibleForTesting
+ UUID getLeaderSessionId() {
+ return leaderSessionId;
+ }
+
+ // ------------------------------------------------------------------------
+ // Internal methods
+ // ------------------------------------------------------------------------
+
+ private void clearState() {
+ jobManagerRegistrations.clear();
+ taskExecutors.clear();
+ slotManager.clearState();
+
+ try {
+ jobLeaderIdService.clear();
+ } catch (Exception e) {
+ onFatalError(new ResourceManagerException("Could not properly clear the job leader id service.", e));
+ }
+
+ leaderSessionId = null;
+ }
+
+ /**
+ * Disconnects the job manager which is connected for the given job from the resource manager.
+ *
+ * @param jobId identifying the job whose leader shall be disconnected
+ */
+ protected void disconnectJobManager(JobID jobId, Exception cause) {
+ JobManagerRegistration jobManagerRegistration = jobManagerRegistrations.remove(jobId);
+
+ if (jobManagerRegistration != null) {
+ log.info("Disconnect job manager {}@{} for job {} from the resource manager.",
+ jobManagerRegistration.getLeaderID(),
+ jobManagerRegistration.getJobManagerGateway().getAddress(),
+ jobId);
+
+ JobMasterGateway jobMasterGateway = jobManagerRegistration.getJobManagerGateway();
+
+ // tell the job manager about the disconnect
+ jobMasterGateway.disconnectResourceManager(jobManagerRegistration.getLeaderID(), getLeaderSessionId(), cause);
+ } else {
+ log.debug("There was no registered job manager for job {}.", jobId);
+ }
+ }
+
+ /**
+ * Checks whether the given resource manager leader id is matching the current leader id and
+ * not null.
+ *
+ * @param resourceManagerLeaderId to check
+ * @return True if the given leader id matches the actual leader id and is not null; otherwise false
+ */
+ protected boolean isValid(UUID resourceManagerLeaderId) {
+ if (resourceManagerLeaderId == null) {
+ return false;
+ } else {
+ return resourceManagerLeaderId.equals(leaderSessionId);
+ }
+ }
+
+ protected void removeJob(JobID jobId) {
+ try {
+ jobLeaderIdService.removeJob(jobId);
+ } catch (Exception e) {
+ log.warn("Could not properly remove the job {} from the job leader id service.", jobId, e);
+ }
+
+ if (jobManagerRegistrations.containsKey(jobId)) {
+ disconnectJobManager(jobId, new Exception("Job " + jobId + "was removed"));
+ }
+ }
+
+ protected void jobLeaderLostLeadership(JobID jobId, UUID oldJobLeaderId) {
+ if (jobManagerRegistrations.containsKey(jobId)) {
+ JobManagerRegistration jobManagerRegistration = jobManagerRegistrations.get(jobId);
+
+ if (jobManagerRegistration.getLeaderID().equals(oldJobLeaderId)) {
+ disconnectJobManager(jobId, new Exception("Job leader lost leadership."));
+ } else {
+ log.debug("Discarding job leader lost leadership, because a new job leader was found for job {}. ", jobId);
+ }
+ } else {
+ log.debug("Discard job leader lost leadership for outdated leader {} for job {}.", oldJobLeaderId, jobId);
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Info messaging
+ // ------------------------------------------------------------------------
+
+ public void sendInfoMessage(final String message) {
+ getRpcService().execute(new Runnable() {
+ @Override
+ public void run() {
+ InfoMessage infoMessage = new InfoMessage(message);
+ for (InfoMessageListenerRpcGateway listenerRpcGateway : infoMessageListeners.values()) {
+ listenerRpcGateway
+ .notifyInfoMessage(infoMessage);
+ }
+ }
+ });
+ }
+
+ // ------------------------------------------------------------------------
+ // Error Handling
+ // ------------------------------------------------------------------------
+
+ /**
+ * Notifies the ResourceManager that a fatal error has occurred and it cannot proceed.
+ * This method should be used when asynchronous threads want to notify the
+ * ResourceManager of a fatal error.
+ *
+ * @param t The exception describing the fatal error
+ */
+ void onFatalErrorAsync(final Throwable t) {
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ onFatalError(t);
+ }
+ });
+ }
+
+ /**
+ * Notifies the ResourceManager that a fatal error has occurred and it cannot proceed.
+ * This method must only be called from within the ResourceManager's main thread.
+ *
+ * @param t The exception describing the fatal error
+ */
+ void onFatalError(Throwable t) {
+ log.error("Fatal error occurred.", t);
+ fatalErrorHandler.onFatalError(t);
+ }
+
+ // ------------------------------------------------------------------------
// Leader Contender
// ------------------------------------------------------------------------
@@ -487,11 +657,11 @@ public abstract class ResourceManager<WorkerType extends Serializable>
log.info("ResourceManager {} was granted leadership with leader session ID {}", getAddress(), newLeaderSessionID);
// clear the state if we've been the leader before
- if (leaderSessionID != null) {
+ if (leaderSessionId != null) {
clearState();
}
- leaderSessionID = newLeaderSessionID;
+ leaderSessionId = newLeaderSessionID;
getRpcService().execute(new Runnable() {
@Override
@@ -516,7 +686,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
clearState();
- leaderSessionID = null;
+ leaderSessionId = null;
}
});
}
@@ -556,82 +726,15 @@ public abstract class ResourceManager<WorkerType extends Serializable>
}
// ------------------------------------------------------------------------
- // Error Handling
- // ------------------------------------------------------------------------
-
- /**
- * Notifies the ResourceManager that a fatal error has occurred and it cannot proceed.
- * This method should be used when asynchronous threads want to notify the
- * ResourceManager of a fatal error.
- *
- * @param t The exception describing the fatal error
- */
- void onFatalErrorAsync(final Throwable t) {
- runAsync(new Runnable() {
- @Override
- public void run() {
- onFatalError(t);
- }
- });
- }
-
- /**
- * Notifies the ResourceManager that a fatal error has occurred and it cannot proceed.
- * This method must only be called from within the ResourceManager's main thread.
- *
- * @param t The exception describing the fatal error
- */
- void onFatalError(Throwable t) {
- log.error("Fatal error occurred.", t);
- fatalErrorHandler.onFatalError(t);
- }
-
- // ------------------------------------------------------------------------
- // Testing methods
- // ------------------------------------------------------------------------
-
- /**
- * Gets the leader session id of current resourceManager.
- *
- * @return return the leaderSessionId of current resourceManager, this returns null until the current resourceManager is granted leadership.
- */
- @VisibleForTesting
- UUID getLeaderSessionID() {
- return leaderSessionID;
- }
-
- // ------------------------------------------------------------------------
- // Internal methods
- // ------------------------------------------------------------------------
-
- private void clearState() {
- jobMasters.clear();
- taskExecutors.clear();
- slotManager.clearState();
- Iterator<JobIdLeaderListener> leaderListenerIterator =
- leaderListeners.values().iterator();
- while (leaderListenerIterator.hasNext()) {
- JobIdLeaderListener listener = leaderListenerIterator.next();
- try {
- listener.stopService();
- } catch (Exception e) {
- onFatalError(e);
- }
- leaderListenerIterator.remove();
- }
- leaderSessionID = new UUID(0, 0);
- }
-
- // ------------------------------------------------------------------------
// Framework specific behavior
// ------------------------------------------------------------------------
/**
* Initializes the framework specific components.
*
- * @throws Exception Exceptions during initialization cause the resource manager to fail.
+ * @throws ResourceManagerException which occurs during initialization and causes the resource manager to fail.
*/
- protected abstract void initialize() throws Exception;
+ protected abstract void initialize() throws ResourceManagerException;
/**
* The framework specific code for shutting down the application. This should report the
@@ -659,24 +762,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
protected abstract WorkerType workerStarted(ResourceID resourceID);
// ------------------------------------------------------------------------
- // Info messaging
- // ------------------------------------------------------------------------
-
- public void sendInfoMessage(final String message) {
- getRpcService().execute(new Runnable() {
- @Override
- public void run() {
- InfoMessage infoMessage = new InfoMessage(message);
- for (InfoMessageListenerRpcGateway listenerRpcGateway : infoMessageListeners.values()) {
- listenerRpcGateway
- .notifyInfoMessage(infoMessage);
- }
- }
- });
- }
-
- // ------------------------------------------------------------------------
- // Resource Manager Services
+ // Static utility classes
// ------------------------------------------------------------------------
protected ResourceManagerServices createResourceManagerServices() {
@@ -687,7 +773,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
@Override
public UUID getLeaderID() {
- return ResourceManager.this.leaderSessionID;
+ return ResourceManager.this.leaderSessionId;
}
@Override
@@ -706,76 +792,26 @@ public abstract class ResourceManager<WorkerType extends Serializable>
}
}
- /**
- * Leader instantiated for each connected JobMaster
- */
- private class JobIdLeaderListener implements LeaderRetrievalListener {
-
- private final JobID jobID;
- private final LeaderRetrievalService retrievalService;
-
- private final FlinkCompletableFuture<UUID> initialLeaderIdFuture;
-
- private volatile UUID leaderID;
-
- private JobIdLeaderListener(
- JobID jobID,
- LeaderRetrievalService retrievalService) throws Exception {
- this.jobID = jobID;
- this.retrievalService = retrievalService;
- this.initialLeaderIdFuture = new FlinkCompletableFuture<>();
- this.retrievalService.start(this);
- }
+ private class JobLeaderIdActionsImpl implements JobLeaderIdActions {
- public Future<UUID> getLeaderID() {
- if (!initialLeaderIdFuture.isDone()) {
- return initialLeaderIdFuture;
- } else {
- return FlinkCompletableFuture.completed(leaderID);
- }
- }
-
- public JobID getJobID() {
- return jobID;
- }
-
-
- public void stopService() throws Exception {
- retrievalService.stop();
+ @Override
+ public void jobLeaderLostLeadership(final JobID jobId, final UUID oldJobLeaderId) {
+ ResourceManager.this.jobLeaderLostLeadership(jobId, oldJobLeaderId);
}
@Override
- public void notifyLeaderAddress(final String leaderAddress, final UUID leaderSessionID) {
- this.leaderID = leaderSessionID;
-
- if (!initialLeaderIdFuture.isDone()) {
- initialLeaderIdFuture.complete(leaderSessionID);
- }
-
- ResourceManager.this.runAsync(new Runnable() {
+ public void removeJob(final JobID jobId) {
+ runAsync(new Runnable() {
@Override
public void run() {
- JobMasterRegistration jobMasterRegistration = ResourceManager.this.jobMasters.get(jobID);
- if (jobMasterRegistration == null || !jobMasterRegistration.getLeaderID().equals(leaderSessionID)) {
- // registration is not valid anymore, remove registration
- ResourceManager.this.jobMasters.remove(jobID);
- // leader listener is not necessary anymore
- JobIdLeaderListener listener = ResourceManager.this.leaderListeners.remove(jobID);
- if (listener != null) {
- try {
- listener.stopService();
- } catch (Exception e) {
- ResourceManager.this.handleError(e);
- }
- }
- }
+ ResourceManager.this.removeJob(jobId);
}
});
}
@Override
- public void handleError(final Exception exception) {
- ResourceManager.this.handleError(exception);
+ public void handleError(Throwable error) {
+ onFatalErrorAsync(error);
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
index 968eeb8..0a37bb9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
@@ -49,7 +49,7 @@ public interface ResourceManagerGateway extends RpcGateway {
* @param timeout Timeout for the future to complete
* @return Future registration response
*/
- Future<RegistrationResponse> registerJobMaster(
+ Future<RegistrationResponse> registerJobManager(
UUID resourceManagerLeaderId,
UUID jobMasterLeaderId,
String jobMasterAddress,
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
index 926be0d..73c8a2d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
@@ -23,6 +23,7 @@ import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcService;
@@ -41,6 +42,7 @@ public class StandaloneResourceManager extends ResourceManager<ResourceID> {
HighAvailabilityServices highAvailabilityServices,
SlotManagerFactory slotManagerFactory,
MetricRegistry metricRegistry,
+ JobLeaderIdService jobLeaderIdService,
FatalErrorHandler fatalErrorHandler) {
super(
rpcService,
@@ -48,11 +50,12 @@ public class StandaloneResourceManager extends ResourceManager<ResourceID> {
highAvailabilityServices,
slotManagerFactory,
metricRegistry,
+ jobLeaderIdService,
fatalErrorHandler);
}
@Override
- protected void initialize() throws Exception {
+ protected void initialize() throws ResourceManagerException {
// nothing to initialize
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerRunner.java
index 1e6f04c..0c7e4e4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerRunner.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.resourcemanager.exceptions;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.resourcemanager.JobLeaderIdService;
import org.apache.flink.runtime.resourcemanager.ResourceManager;
import org.apache.flink.runtime.resourcemanager.ResourceManagerConfiguration;
import org.apache.flink.runtime.resourcemanager.StandaloneResourceManager;
@@ -48,7 +49,7 @@ public class ResourceManagerRunner implements FatalErrorHandler {
final Configuration configuration,
final RpcService rpcService,
final HighAvailabilityServices highAvailabilityServices,
- final MetricRegistry metricRegistry) throws ConfigurationException {
+ final MetricRegistry metricRegistry) throws Exception {
Preconditions.checkNotNull(configuration);
Preconditions.checkNotNull(rpcService);
@@ -57,6 +58,7 @@ public class ResourceManagerRunner implements FatalErrorHandler {
final ResourceManagerConfiguration resourceManagerConfiguration = ResourceManagerConfiguration.fromConfiguration(configuration);
final SlotManagerFactory slotManagerFactory = new DefaultSlotManager.Factory();
+ final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(highAvailabilityServices);
this.resourceManager = new StandaloneResourceManager(
rpcService,
@@ -64,6 +66,7 @@ public class ResourceManagerRunner implements FatalErrorHandler {
highAvailabilityServices,
slotManagerFactory,
metricRegistry,
+ jobLeaderIdService,
this);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobManagerRegistration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobManagerRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobManagerRegistration.java
new file mode 100644
index 0000000..a1deb65
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobManagerRegistration.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.registration;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.jobmaster.JobMasterGateway;
+import org.apache.flink.util.Preconditions;
+
+import java.util.UUID;
+
+/**
+ * Container for JobManager related registration information, such as the leader id or the job id.
+ */
+public class JobManagerRegistration {
+ private final JobID jobID;
+
+ private final UUID leaderID;
+
+ private final JobMasterGateway jobManagerGateway;
+
+ public JobManagerRegistration(
+ JobID jobID,
+ UUID leaderID,
+ JobMasterGateway jobManagerGateway) {
+ this.jobID = Preconditions.checkNotNull(jobID);
+ this.leaderID = Preconditions.checkNotNull(leaderID);
+ this.jobManagerGateway = Preconditions.checkNotNull(jobManagerGateway);
+ }
+
+ public JobID getJobID() {
+ return jobID;
+ }
+
+ public UUID getLeaderID() {
+ return leaderID;
+ }
+
+ public JobMasterGateway getJobManagerGateway() {
+ return jobManagerGateway;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobMasterRegistration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobMasterRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobMasterRegistration.java
deleted file mode 100644
index f417935..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobMasterRegistration.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.resourcemanager.registration;
-
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.jobmaster.JobMasterGateway;
-
-import java.util.UUID;
-
-/**
- * This class is responsible for grouping the JobMasterGateway and the JobMaster's
- * leader id
- */
-public class JobMasterRegistration {
-
- private static final long serialVersionUID = -2062957799469434614L;
-
- private final JobID jobID;
-
- private final UUID leaderID;
-
- private final JobMasterGateway jobMasterGateway;
-
- public JobMasterRegistration(
- JobID jobID,
- UUID leaderID,
- JobMasterGateway jobMasterGateway) {
- this.jobID = jobID;
- this.leaderID = leaderID;
- this.jobMasterGateway = jobMasterGateway;
- }
-
- public JobID getJobID() {
- return jobID;
- }
-
-
- public UUID getLeaderID() {
- return leaderID;
- }
-
- public JobMasterGateway getJobMasterGateway() {
- return jobMasterGateway;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
index 44719c8..6e3fb40 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
@@ -96,8 +96,6 @@ public class AkkaRpcService implements RpcService {
Address actorSystemAddress = AkkaUtils.getAddress(actorSystem);
-
-
if (actorSystemAddress.host().isDefined()) {
address = actorSystemAddress.host().get();
} else {
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
index cb38e6e..2e52eeb 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
@@ -49,6 +49,7 @@ public class ResourceManagerHATest {
ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L));
SlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
MetricRegistry metricRegistry = mock(MetricRegistry.class);
+ JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(highAvailabilityServices);
TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
final ResourceManager resourceManager =
@@ -58,17 +59,18 @@ public class ResourceManagerHATest {
highAvailabilityServices,
slotManagerFactory,
metricRegistry,
+ jobLeaderIdService,
testingFatalErrorHandler);
resourceManager.start();
// before grant leadership, resourceManager's leaderId is null
- Assert.assertEquals(null, resourceManager.getLeaderSessionID());
+ Assert.assertEquals(null, resourceManager.getLeaderSessionId());
final UUID leaderId = UUID.randomUUID();
leaderElectionService.isLeader(leaderId);
// after grant leadership, resourceManager's leaderId has value
- Assert.assertEquals(leaderId, resourceManager.getLeaderSessionID());
+ Assert.assertEquals(leaderId, resourceManager.getLeaderSessionId());
// then revoke leadership, resourceManager's leaderId is null again
leaderElectionService.notLeader();
- Assert.assertEquals(null, resourceManager.getLeaderSessionID());
+ Assert.assertEquals(null, resourceManager.getLeaderSessionId());
if (testingFatalErrorHandler.hasExceptionOccurred()) {
testingFatalErrorHandler.rethrowError();
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
index 7b8d254..2622634 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
@@ -71,8 +71,8 @@ public class ResourceManagerJobMasterTest {
final UUID rmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
// test response successful
- Future<RegistrationResponse> successfulFuture = resourceManager.registerJobMaster(rmLeaderSessionId, jmLeaderID, jobMasterAddress, jobID);
- RegistrationResponse response = successfulFuture.get(5, TimeUnit.SECONDS);
+ Future<RegistrationResponse> successfulFuture = resourceManager.registerJobManager(rmLeaderSessionId, jmLeaderID, jobMasterAddress, jobID);
+ RegistrationResponse response = successfulFuture.get(5L, TimeUnit.SECONDS);
assertTrue(response instanceof JobMasterRegistrationSuccess);
if (testingFatalErrorHandler.hasExceptionOccurred()) {
@@ -96,7 +96,7 @@ public class ResourceManagerJobMasterTest {
// test throw exception when receive a registration from job master which takes unmatched leaderSessionId
UUID differentLeaderSessionID = UUID.randomUUID();
- Future<RegistrationResponse> unMatchedLeaderFuture = resourceManager.registerJobMaster(differentLeaderSessionID, jmLeaderID, jobMasterAddress, jobID);
+ Future<RegistrationResponse> unMatchedLeaderFuture = resourceManager.registerJobManager(differentLeaderSessionID, jmLeaderID, jobMasterAddress, jobID);
assertTrue(unMatchedLeaderFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
if (testingFatalErrorHandler.hasExceptionOccurred()) {
@@ -120,7 +120,7 @@ public class ResourceManagerJobMasterTest {
// test throw exception when receive a registration from job master which takes unmatched leaderSessionId
UUID differentLeaderSessionID = UUID.randomUUID();
- Future<RegistrationResponse> unMatchedLeaderFuture = resourceManager.registerJobMaster(rmLeaderSessionId, differentLeaderSessionID, jobMasterAddress, jobID);
+ Future<RegistrationResponse> unMatchedLeaderFuture = resourceManager.registerJobManager(rmLeaderSessionId, differentLeaderSessionID, jobMasterAddress, jobID);
assertTrue(unMatchedLeaderFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
if (testingFatalErrorHandler.hasExceptionOccurred()) {
@@ -144,7 +144,7 @@ public class ResourceManagerJobMasterTest {
// test throw exception when receive a registration from job master which takes invalid address
String invalidAddress = "/jobMasterAddress2";
- Future<RegistrationResponse> invalidAddressFuture = resourceManager.registerJobMaster(rmLeaderSessionId, jmLeaderSessionId, invalidAddress, jobID);
+ Future<RegistrationResponse> invalidAddressFuture = resourceManager.registerJobManager(rmLeaderSessionId, jmLeaderSessionId, invalidAddress, jobID);
assertTrue(invalidAddressFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
if (testingFatalErrorHandler.hasExceptionOccurred()) {
@@ -168,7 +168,7 @@ public class ResourceManagerJobMasterTest {
JobID unknownJobIDToHAServices = new JobID();
// verify return RegistrationResponse.Decline when failed to start a job master Leader retrieval listener
- Future<RegistrationResponse> declineFuture = resourceManager.registerJobMaster(rmLeaderSessionId, jmLeaderSessionId, jobMasterAddress, unknownJobIDToHAServices);
+ Future<RegistrationResponse> declineFuture = resourceManager.registerJobManager(rmLeaderSessionId, jmLeaderSessionId, jobMasterAddress, unknownJobIDToHAServices);
RegistrationResponse response = declineFuture.get(5, TimeUnit.SECONDS);
assertTrue(response instanceof RegistrationResponse.Decline);
@@ -196,6 +196,7 @@ public class ResourceManagerJobMasterTest {
ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L));
SlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
MetricRegistry metricRegistry = mock(MetricRegistry.class);
+ JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(highAvailabilityServices);
ResourceManager resourceManager = new StandaloneResourceManager(
rpcService,
@@ -203,6 +204,7 @@ public class ResourceManagerJobMasterTest {
highAvailabilityServices,
slotManagerFactory,
metricRegistry,
+ jobLeaderIdService,
fatalErrorHandler);
resourceManager.start();
return resourceManager;
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
index 4640eab..1016181 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
@@ -148,6 +148,7 @@ public class ResourceManagerTaskExecutorTest {
TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L));
MetricRegistry metricRegistry = mock(MetricRegistry.class);
+ JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(highAvailabilityServices);
StandaloneResourceManager resourceManager =
@@ -157,6 +158,7 @@ public class ResourceManagerTaskExecutorTest {
highAvailabilityServices,
slotManagerFactory,
metricRegistry,
+ jobLeaderIdService,
fatalErrorHandler);
resourceManager.start();
return resourceManager;
http://git-wip-us.apache.org/repos/asf/flink/blob/2a5a5b39/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
index 08ceb86..a3ba436 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
@@ -32,6 +32,7 @@ import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.registration.RegistrationResponse;
+import org.apache.flink.runtime.resourcemanager.JobLeaderIdService;
import org.apache.flink.runtime.resourcemanager.ResourceManager;
import org.apache.flink.runtime.resourcemanager.ResourceManagerConfiguration;
import org.apache.flink.runtime.resourcemanager.ResourceManagerServices;
@@ -107,6 +108,7 @@ public class SlotProtocolTest extends TestLogger {
configureHA(testingHaServices, jobID, rmAddress, rmLeaderID, jmAddress, jmLeaderID);
ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L));
+ JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(testingHaServices);
final TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
SpiedResourceManager resourceManager =
@@ -116,12 +118,13 @@ public class SlotProtocolTest extends TestLogger {
testingHaServices,
slotManagerFactory,
mock(MetricRegistry.class),
+ jobLeaderIdService,
mock(FatalErrorHandler.class));
resourceManager.start();
rmLeaderElectionService.isLeader(rmLeaderID);
Future<RegistrationResponse> registrationFuture =
- resourceManager.registerJobMaster(rmLeaderID, jmLeaderID, jmAddress, jobID);
+ resourceManager.registerJobManager(rmLeaderID, jmLeaderID, jmAddress, jobID);
try {
registrationFuture.get(5, TimeUnit.SECONDS);
} catch (Exception e) {
@@ -207,6 +210,8 @@ public class SlotProtocolTest extends TestLogger {
ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L));
+ JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(testingHaServices);
+
TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
ResourceManager<ResourceID> resourceManager =
Mockito.spy(new StandaloneResourceManager(
@@ -215,6 +220,7 @@ public class SlotProtocolTest extends TestLogger {
testingHaServices,
slotManagerFactory,
mock(MetricRegistry.class),
+ jobLeaderIdService,
mock(FatalErrorHandler.class)));
resourceManager.start();
rmLeaderElectionService.isLeader(rmLeaderID);
@@ -222,7 +228,7 @@ public class SlotProtocolTest extends TestLogger {
Thread.sleep(1000);
Future<RegistrationResponse> registrationFuture =
- resourceManager.registerJobMaster(rmLeaderID, jmLeaderID, jmAddress, jobID);
+ resourceManager.registerJobManager(rmLeaderID, jmLeaderID, jmAddress, jobID);
try {
registrationFuture.get(5L, TimeUnit.SECONDS);
} catch (Exception e) {
@@ -290,6 +296,7 @@ public class SlotProtocolTest extends TestLogger {
HighAvailabilityServices highAvailabilityServices,
SlotManagerFactory slotManagerFactory,
MetricRegistry metricRegistry,
+ JobLeaderIdService jobLeaderIdService,
FatalErrorHandler fatalErrorHandler) {
super(
rpcService,
@@ -297,6 +304,7 @@ public class SlotProtocolTest extends TestLogger {
highAvailabilityServices,
slotManagerFactory,
metricRegistry,
+ jobLeaderIdService,
fatalErrorHandler);
}
[28/50] [abbrv] flink git commit: [FLINK-4375] [distributed
coordination] Implement new JobManager creation, initialization,
and basic RPC methods
Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 3b8fc97..e11f3a1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -18,38 +18,32 @@
package org.apache.flink.runtime.jobmaster;
-import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.time.Time;
-import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.io.InputSplit;
import org.apache.flink.core.io.InputSplitAssigner;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
import org.apache.flink.runtime.checkpoint.CheckpointCoordinator;
-import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
+import org.apache.flink.runtime.checkpoint.CheckpointMetaData;
import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
-import org.apache.flink.runtime.checkpoint.CompletedCheckpointStore;
-import org.apache.flink.runtime.checkpoint.savepoint.SavepointStore;
-import org.apache.flink.runtime.checkpoint.stats.CheckpointStatsTracker;
-import org.apache.flink.runtime.checkpoint.stats.DisabledCheckpointStatsTracker;
-import org.apache.flink.runtime.checkpoint.stats.SimpleCheckpointStatsTracker;
+import org.apache.flink.runtime.checkpoint.CompletedCheckpoint;
import org.apache.flink.runtime.client.JobExecutionException;
import org.apache.flink.runtime.client.JobSubmissionException;
import org.apache.flink.runtime.client.SerializedJobExecutionResult;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.concurrent.BiFunction;
+import org.apache.flink.runtime.concurrent.ApplyFunction;
import org.apache.flink.runtime.concurrent.Future;
-import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
-import org.apache.flink.runtime.concurrent.impl.FlinkFuture;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager;
import org.apache.flink.runtime.executiongraph.Execution;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+import org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder;
import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+import org.apache.flink.runtime.executiongraph.JobStatusListener;
import org.apache.flink.runtime.executiongraph.restart.RestartStrategy;
import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
@@ -59,18 +53,13 @@ import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobStatus;
-import org.apache.flink.runtime.jobgraph.JobVertex;
import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.jobgraph.jsonplan.JsonPlanGenerator;
-import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
import org.apache.flink.runtime.jobmanager.OnCompletionActions;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
import org.apache.flink.runtime.jobmaster.message.ClassloadingProps;
-import org.apache.flink.runtime.jobmaster.message.DisposeSavepointResponse;
-import org.apache.flink.runtime.jobmaster.message.NextInputSplit;
-import org.apache.flink.runtime.jobmaster.message.TriggerSavepointResponse;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint;
import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup;
@@ -83,25 +72,27 @@ import org.apache.flink.runtime.registration.RegisteredRpcConnection;
import org.apache.flink.runtime.registration.RegistrationResponse;
import org.apache.flink.runtime.registration.RetryingRegistration;
import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.runtime.rpc.StartStoppable;
+import org.apache.flink.runtime.state.CheckpointStateHandles;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import org.apache.flink.runtime.util.SerializedThrowable;
import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.SerializedValue;
-import org.slf4j.Logger;
-import scala.concurrent.ExecutionContext$;
-import scala.concurrent.duration.FiniteDuration;
+import org.slf4j.Logger;
+import javax.annotation.Nullable;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -112,16 +103,21 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
* It offers the following methods as part of its rpc interface to interact with the JobMaster
* remotely:
* <ul>
- * <li>{@link #updateTaskExecutionState(TaskExecutionState)} updates the task execution state for
+ * <li>{@link #updateTaskExecutionState} updates the task execution state for
* given task</li>
* </ul>
*/
public class JobMaster extends RpcEndpoint<JobMasterGateway> {
+ private static final AtomicReferenceFieldUpdater<JobMaster, UUID> LEADER_ID_UPDATER =
+ AtomicReferenceFieldUpdater.newUpdater(JobMaster.class, UUID.class, "leaderSessionID");
+
+ // ------------------------------------------------------------------------
+
/** Logical representation of the job */
private final JobGraph jobGraph;
- /** Configuration of the job */
+ /** Configuration of the JobManager */
private final Configuration configuration;
/** Service to contend for and retrieve the leadership of JM and RM */
@@ -130,37 +126,24 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
/** Blob cache manager used across jobs */
private final BlobLibraryCacheManager libraryCacheManager;
- /** Factory to create restart strategy for this job */
- private final RestartStrategyFactory restartStrategyFactory;
-
- /** Store for save points */
- private final SavepointStore savepointStore;
-
- /** The timeout for this job */
- private final Time timeout;
-
- /** The scheduler to use for scheduling new tasks as they are needed */
- private final Scheduler scheduler;
+ /** The metrics for the JobManager itself */
+ private final MetricGroup jobManagerMetricGroup;
- /** The metrics group used across jobs */
- private final JobManagerMetricGroup jobManagerMetricGroup;
+ /** The metrics for the job */
+ private final MetricGroup jobMetricGroup;
/** The execution context which is used to execute futures */
- private final Executor executionContext;
+ private final ExecutorService executionContext;
private final OnCompletionActions jobCompletionActions;
- /** The execution graph of this job */
- private volatile ExecutionGraph executionGraph;
-
- /** The checkpoint recovery factory used by this job */
- private CheckpointRecoveryFactory checkpointRecoveryFactory;
+ private final FatalErrorHandler errorHandler;
- private ClassLoader userCodeLoader;
+ private final ClassLoader userCodeLoader;
- private RestartStrategy restartStrategy;
+ /** The execution graph of this job */
+ private final ExecutionGraph executionGraph;
- private MetricGroup jobMetrics;
private volatile UUID leaderSessionID;
@@ -170,22 +153,26 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
private LeaderRetrievalService resourceManagerLeaderRetriever;
/** Connection with ResourceManager, null if not located address yet or we close it initiative */
- private volatile ResourceManagerConnection resourceManagerConnection;
+ private ResourceManagerConnection resourceManagerConnection;
+
+ // TODO - we need to replace this with the slot pool
+ private final Scheduler scheduler;
// ------------------------------------------------------------------------
public JobMaster(
- JobGraph jobGraph,
- Configuration configuration,
- RpcService rpcService,
- HighAvailabilityServices highAvailabilityService,
- BlobLibraryCacheManager libraryCacheManager,
- RestartStrategyFactory restartStrategyFactory,
- SavepointStore savepointStore,
- Time timeout,
- Scheduler scheduler,
- JobManagerMetricGroup jobManagerMetricGroup,
- OnCompletionActions jobCompletionActions)
+ JobGraph jobGraph,
+ Configuration configuration,
+ RpcService rpcService,
+ HighAvailabilityServices highAvailabilityService,
+ ExecutorService executorService,
+ BlobLibraryCacheManager libraryCacheManager,
+ RestartStrategyFactory restartStrategyFactory,
+ Time rpcAskTimeout,
+ @Nullable JobManagerMetricGroup jobManagerMetricGroup,
+ OnCompletionActions jobCompletionActions,
+ FatalErrorHandler errorHandler,
+ ClassLoader userCodeLoader) throws Exception
{
super(rpcService);
@@ -193,289 +180,149 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
this.configuration = checkNotNull(configuration);
this.highAvailabilityServices = checkNotNull(highAvailabilityService);
this.libraryCacheManager = checkNotNull(libraryCacheManager);
- this.restartStrategyFactory = checkNotNull(restartStrategyFactory);
- this.savepointStore = checkNotNull(savepointStore);
- this.timeout = checkNotNull(timeout);
- this.scheduler = checkNotNull(scheduler);
- this.jobManagerMetricGroup = checkNotNull(jobManagerMetricGroup);
- this.executionContext = checkNotNull(rpcService.getExecutor());
+ this.executionContext = checkNotNull(executorService);
this.jobCompletionActions = checkNotNull(jobCompletionActions);
- }
+ this.errorHandler = checkNotNull(errorHandler);
+ this.userCodeLoader = checkNotNull(userCodeLoader);
- //----------------------------------------------------------------------------------------------
- // Lifecycle management
- //----------------------------------------------------------------------------------------------
-
- /**
- * Initializing the job execution environment, should be called before start. Any error occurred during
- * initialization will be treated as job submission failure.
- *
- * @throws JobSubmissionException
- */
- public void init() throws JobSubmissionException {
- log.info("Initializing job {} ({}).", jobGraph.getName(), jobGraph.getJobID());
+ final String jobName = jobGraph.getName();
+ final JobID jid = jobGraph.getJobID();
- try {
- // IMPORTANT: We need to make sure that the library registration is the first action,
- // because this makes sure that the uploaded jar files are removed in case of
- // unsuccessful
- try {
- libraryCacheManager.registerJob(jobGraph.getJobID(), jobGraph.getUserJarBlobKeys(),
- jobGraph.getClasspaths());
- } catch (Throwable t) {
- throw new JobSubmissionException(jobGraph.getJobID(),
- "Cannot set up the user code libraries: " + t.getMessage(), t);
- }
+ if (jobManagerMetricGroup != null) {
+ this.jobManagerMetricGroup = jobManagerMetricGroup;
+ this.jobMetricGroup = jobManagerMetricGroup.addJob(jobGraph);
+ } else {
+ this.jobManagerMetricGroup = new UnregisteredMetricsGroup();
+ this.jobMetricGroup = new UnregisteredMetricsGroup();
+ }
- userCodeLoader = libraryCacheManager.getClassLoader(jobGraph.getJobID());
- if (userCodeLoader == null) {
- throw new JobSubmissionException(jobGraph.getJobID(),
- "The user code class loader could not be initialized.");
- }
+ log.info("Initializing job {} ({}).", jobName, jid);
- if (jobGraph.getNumberOfVertices() == 0) {
- throw new JobSubmissionException(jobGraph.getJobID(), "The given job is empty");
- }
-
- final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration =
+ final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration =
jobGraph.getSerializedExecutionConfig()
- .deserializeValue(userCodeLoader)
- .getRestartStrategy();
- if (restartStrategyConfiguration != null) {
- restartStrategy = RestartStrategyFactory.createRestartStrategy(restartStrategyConfiguration);
- }
- else {
- restartStrategy = restartStrategyFactory.createRestartStrategy();
- }
+ .deserializeValue(userCodeLoader)
+ .getRestartStrategy();
- log.info("Using restart strategy {} for {} ({}).", restartStrategy, jobGraph.getName(), jobGraph.getJobID());
+ final RestartStrategy restartStrategy = (restartStrategyConfiguration != null) ?
+ RestartStrategyFactory.createRestartStrategy(restartStrategyConfiguration) :
+ restartStrategyFactory.createRestartStrategy();
- if (jobManagerMetricGroup != null) {
- jobMetrics = jobManagerMetricGroup.addJob(jobGraph);
- }
- if (jobMetrics == null) {
- jobMetrics = new UnregisteredMetricsGroup();
- }
+ log.info("Using restart strategy {} for {} ({}).", restartStrategy, jobName, jid);
- try {
- checkpointRecoveryFactory = highAvailabilityServices.getCheckpointRecoveryFactory();
- } catch (Exception e) {
- log.error("Could not get the checkpoint recovery factory.", e);
- throw new JobSubmissionException(jobGraph.getJobID(), "Could not get the checkpoint recovery factory.", e);
- }
+ CheckpointRecoveryFactory checkpointRecoveryFactory;
+ try {
+ checkpointRecoveryFactory = highAvailabilityServices.getCheckpointRecoveryFactory();
+ } catch (Exception e) {
+ log.error("Could not create the access to highly-available checkpoint storage.", e);
+ throw new Exception("Could not create the access to highly-available checkpoint storage.", e);
+ }
- try {
- resourceManagerLeaderRetriever = highAvailabilityServices.getResourceManagerLeaderRetriever();
- } catch (Exception e) {
- log.error("Could not get the resource manager leader retriever.", e);
- throw new JobSubmissionException(jobGraph.getJobID(),
+ try {
+ resourceManagerLeaderRetriever = highAvailabilityServices.getResourceManagerLeaderRetriever();
+ } catch (Exception e) {
+ log.error("Could not get the resource manager leader retriever.", e);
+ throw new JobSubmissionException(jobGraph.getJobID(),
"Could not get the resource manager leader retriever.", e);
- }
- } catch (Throwable t) {
- log.error("Failed to initializing job {} ({})", jobGraph.getName(), jobGraph.getJobID(), t);
+ }
- libraryCacheManager.unregisterJob(jobGraph.getJobID());
+ this.executionGraph = ExecutionGraphBuilder.buildGraph(
+ null,
+ jobGraph,
+ configuration,
+ executorService,
+ userCodeLoader,
+ checkpointRecoveryFactory,
+ rpcAskTimeout,
+ restartStrategy,
+ jobMetricGroup,
+ -1,
+ log);
- if (t instanceof JobSubmissionException) {
- throw (JobSubmissionException) t;
- }
- else {
- throw new JobSubmissionException(jobGraph.getJobID(), "Failed to initialize job " +
- jobGraph.getName() + " (" + jobGraph.getJobID() + ")", t);
- }
- }
+ // TODO - temp fix
+ this.scheduler = new Scheduler(executorService);
}
+ //----------------------------------------------------------------------------------------------
+ // Lifecycle management
+ //----------------------------------------------------------------------------------------------
+
+
@Override
public void start() {
- super.start();
+ throw new UnsupportedOperationException("Should never call start() without leader ID");
}
+ /**
+ * Start the rpc service and begin to run the job.
+ *
+ * @param leaderSessionID The necessary leader id for running the job.
+ */
+ public void start(final UUID leaderSessionID) {
+ if (LEADER_ID_UPDATER.compareAndSet(this, null, leaderSessionID)) {
+ super.start();
+
+ log.info("Starting JobManager for job {} ({})", jobGraph.getName(), jobGraph.getJobID());
+ getSelf().startJobExecution();
+ } else {
+ log.warn("Job already started with leaderId {}, ignoring this start request.", leaderSessionID);
+ }
+ }
+
+ /**
+ * Suspend the job and shutdown all other services including rpc.
+ */
@Override
public void shutDown() {
+ // make sure there is a graceful exit
+ getSelf().suspendExecution(new Exception("JobManager is shutting down."));
super.shutDown();
-
- suspendJob(new Exception("JobManager is shutting down."));
-
- disposeCommunicationWithResourceManager();
}
-
-
//----------------------------------------------------------------------------------------------
// RPC methods
//----------------------------------------------------------------------------------------------
- /**
- * Start to run the job, runtime data structures like ExecutionGraph will be constructed now and checkpoint
- * being recovered. After this, we will begin to schedule the job.
- */
+ //-- job starting and stopping -----------------------------------------------------------------
+
@RpcMethod
- public void startJob(final UUID leaderSessionID) {
- log.info("Starting job {} ({}) with leaderId {}.", jobGraph.getName(), jobGraph.getJobID(), leaderSessionID);
-
- this.leaderSessionID = leaderSessionID;
-
- if (executionGraph != null) {
- executionGraph = new ExecutionGraph(
- ExecutionContext$.MODULE$.fromExecutor(executionContext),
- jobGraph.getJobID(),
- jobGraph.getName(),
- jobGraph.getJobConfiguration(),
- jobGraph.getSerializedExecutionConfig(),
- new FiniteDuration(timeout.getSize(), timeout.getUnit()),
- restartStrategy,
- jobGraph.getUserJarBlobKeys(),
- jobGraph.getClasspaths(),
- userCodeLoader,
- jobMetrics);
- }
- else {
- // TODO: update last active time in JobInfo
- }
+ public void startJobExecution() {
+ log.info("Starting execution of job {} ({}) with leaderId {}.",
+ jobGraph.getName(), jobGraph.getJobID(), leaderSessionID);
try {
- executionGraph.setScheduleMode(jobGraph.getScheduleMode());
- executionGraph.setQueuedSchedulingAllowed(jobGraph.getAllowQueuedScheduling());
-
- try {
- executionGraph.setJsonPlan(JsonPlanGenerator.generatePlan(jobGraph));
- } catch (Exception e) {
- log.warn("Cannot create JSON plan for job {} ({})", jobGraph.getJobID(), jobGraph.getName(), e);
- executionGraph.setJsonPlan("{}");
- }
-
- // initialize the vertices that have a master initialization hook
- // file output formats create directories here, input formats create splits
- if (log.isDebugEnabled()) {
- log.debug("Running initialization on master for job {} ({}).", jobGraph.getJobID(), jobGraph.getName());
- }
- for (JobVertex vertex : jobGraph.getVertices()) {
- final String executableClass = vertex.getInvokableClassName();
- if (executableClass == null || executableClass.length() == 0) {
- throw new JobExecutionException(jobGraph.getJobID(),
- "The vertex " + vertex.getID() + " (" + vertex.getName() + ") has no invokable class.");
- }
- if (vertex.getParallelism() == ExecutionConfig.PARALLELISM_AUTO_MAX) {
- vertex.setParallelism(scheduler.getTotalNumberOfSlots());
- }
-
- try {
- vertex.initializeOnMaster(userCodeLoader);
- } catch (Throwable t) {
- throw new JobExecutionException(jobGraph.getJobID(),
- "Cannot initialize task '" + vertex.getName() + "': " + t.getMessage(), t);
- }
- }
-
- // topologically sort the job vertices and attach the graph to the existing one
- final List<JobVertex> sortedTopology = jobGraph.getVerticesSortedTopologicallyFromSources();
- if (log.isDebugEnabled()) {
- log.debug("Adding {} vertices from job graph {} ({}).", sortedTopology.size(),
- jobGraph.getJobID(), jobGraph.getName());
- }
- executionGraph.attachJobGraph(sortedTopology);
-
- if (log.isDebugEnabled()) {
- log.debug("Successfully created execution graph from job graph {} ({}).",
- jobGraph.getJobID(), jobGraph.getName());
- }
-
- final JobSnapshottingSettings snapshotSettings = jobGraph.getSnapshotSettings();
- if (snapshotSettings != null) {
- List<ExecutionJobVertex> triggerVertices = getExecutionJobVertexWithId(
- executionGraph, snapshotSettings.getVerticesToTrigger());
-
- List<ExecutionJobVertex> ackVertices = getExecutionJobVertexWithId(
- executionGraph, snapshotSettings.getVerticesToAcknowledge());
-
- List<ExecutionJobVertex> confirmVertices = getExecutionJobVertexWithId(
- executionGraph, snapshotSettings.getVerticesToConfirm());
-
- CompletedCheckpointStore completedCheckpoints = checkpointRecoveryFactory.createCheckpointStore(
- jobGraph.getJobID(), userCodeLoader);
-
- CheckpointIDCounter checkpointIdCounter = checkpointRecoveryFactory.createCheckpointIDCounter(
- jobGraph.getJobID());
-
- // Checkpoint stats tracker
- boolean isStatsDisabled = configuration.getBoolean(
- ConfigConstants.JOB_MANAGER_WEB_CHECKPOINTS_DISABLE,
- ConfigConstants.DEFAULT_JOB_MANAGER_WEB_CHECKPOINTS_DISABLE);
-
- final CheckpointStatsTracker checkpointStatsTracker;
- if (isStatsDisabled) {
- checkpointStatsTracker = new DisabledCheckpointStatsTracker();
- }
- else {
- int historySize = configuration.getInteger(
- ConfigConstants.JOB_MANAGER_WEB_CHECKPOINTS_HISTORY_SIZE,
- ConfigConstants.DEFAULT_JOB_MANAGER_WEB_CHECKPOINTS_HISTORY_SIZE);
- checkpointStatsTracker = new SimpleCheckpointStatsTracker(historySize, ackVertices, jobMetrics);
- }
-
- executionGraph.enableSnapshotCheckpointing(
- snapshotSettings.getCheckpointInterval(),
- snapshotSettings.getCheckpointTimeout(),
- snapshotSettings.getMinPauseBetweenCheckpoints(),
- snapshotSettings.getMaxConcurrentCheckpoints(),
- triggerVertices,
- ackVertices,
- confirmVertices,
- checkpointIdCounter,
- completedCheckpoints,
- savepointStore,
- checkpointStatsTracker);
- }
-
- // TODO: register this class to execution graph as job status change listeners
-
- // TODO: register client as job / execution status change listeners if they are interested
-
- /*
- TODO: decide whether we should take the savepoint before recovery
-
- if (isRecovery) {
- // this is a recovery of a master failure (this master takes over)
- executionGraph.restoreLatestCheckpointedState();
- } else {
- if (snapshotSettings != null) {
- String savepointPath = snapshotSettings.getSavepointPath();
- if (savepointPath != null) {
- // got a savepoint
- log.info("Starting job from savepoint {}.", savepointPath);
-
- // load the savepoint as a checkpoint into the system
- final CompletedCheckpoint savepoint = SavepointLoader.loadAndValidateSavepoint(
- jobGraph.getJobID(), executionGraph.getAllVertices(), savepointStore, savepointPath);
- executionGraph.getCheckpointCoordinator().getCheckpointStore().addCheckpoint(savepoint);
-
- // Reset the checkpoint ID counter
- long nextCheckpointId = savepoint.getCheckpointID() + 1;
- log.info("Reset the checkpoint ID to " + nextCheckpointId);
- executionGraph.getCheckpointCoordinator().getCheckpointIdCounter().setCount(nextCheckpointId);
-
- executionGraph.restoreLatestCheckpointedState();
- }
+ // register self as job status change listener
+ executionGraph.registerJobStatusListener(new JobStatusListener() {
+ @Override
+ public void jobStatusChanges(
+ final JobID jobId, final JobStatus newJobStatus, final long timestamp, final Throwable error)
+ {
+ // run in rpc thread to avoid concurrency
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ jobStatusChanged(newJobStatus, timestamp, error);
+ }
+ });
}
- }
- */
+ });
- // job is good to go, try to locate resource manager's address
+ // job is ready to go, try to establish connection with resource manager
resourceManagerLeaderRetriever.start(new ResourceManagerLeaderListener());
} catch (Throwable t) {
+
+ // TODO - this should not result in a job failure, but another leader should take over
+ // TODO - either this master should retry the execution, or it should relinquish leadership / terminate
+
log.error("Failed to start job {} ({})", jobGraph.getName(), jobGraph.getJobID(), t);
executionGraph.fail(t);
- executionGraph = null;
- final Throwable rt;
+ final JobExecutionException rt;
if (t instanceof JobExecutionException) {
rt = (JobExecutionException) t;
- }
- else {
+ } else {
rt = new JobExecutionException(jobGraph.getJobID(),
- "Failed to start job " + jobGraph.getJobID() + " (" + jobGraph.getName() + ")", t);
+ "Failed to start job " + jobGraph.getJobID() + " (" + jobGraph.getName() + ")", t);
}
// TODO: notify client about this failure
@@ -488,34 +335,51 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
executionContext.execute(new Runnable() {
@Override
public void run() {
- if (executionGraph != null) {
- try {
- executionGraph.scheduleForExecution(scheduler);
- } catch (Throwable t) {
- executionGraph.fail(t);
- }
+ try {
+ executionGraph.scheduleForExecution(scheduler);
+ } catch (Throwable t) {
+ executionGraph.fail(t);
}
}
});
}
/**
- * Suspending job, all the running tasks will be cancelled, and runtime data will be cleared.
+ * Suspending job, all the running tasks will be cancelled, and communication with other components
+ * will be disposed.
+ *
+ * <p>Mostly job is suspended because of the leadership has been revoked, one can be restart this job by
+ * calling the {@link #start(UUID)} method once we take the leadership back again.
*
* @param cause The reason of why this job been suspended.
*/
@RpcMethod
- public void suspendJob(final Throwable cause) {
+ public void suspendExecution(final Throwable cause) {
+ if (leaderSessionID == null) {
+ log.debug("Job has already been suspended or shutdown.");
+ return;
+ }
+
+ // receive no more messages until started again, should be called before we clear self leader id
+ ((StartStoppable) getSelf()).stop();
+
leaderSessionID = null;
+ executionGraph.suspend(cause);
- if (executionGraph != null) {
- executionGraph.suspend(cause);
- executionGraph = null;
+ // disconnect from resource manager:
+ try {
+ resourceManagerLeaderRetriever.stop();
+ } catch (Exception e) {
+ log.warn("Failed to stop resource manager leader retriever when suspending.");
}
+ closeResourceManagerConnection();
+
+ // TODO: disconnect from all registered task managers
- disposeCommunicationWithResourceManager();
}
+ //----------------------------------------------------------------------------------------------
+
/**
* Updates the task execution state for a given task.
*
@@ -523,24 +387,38 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
* @return Acknowledge the task execution state update
*/
@RpcMethod
- public Acknowledge updateTaskExecutionState(final TaskExecutionState taskExecutionState) throws ExecutionGraphException {
+ public Acknowledge updateTaskExecutionState(
+ final UUID leaderSessionID,
+ final TaskExecutionState taskExecutionState) throws Exception
+ {
if (taskExecutionState == null) {
throw new NullPointerException("TaskExecutionState must not be null.");
}
+ if (!this.leaderSessionID.equals(leaderSessionID)) {
+ throw new Exception("Leader id not match, expected: " + this.leaderSessionID
+ + ", actual: " + leaderSessionID);
+ }
+
if (executionGraph.updateState(taskExecutionState)) {
return Acknowledge.get();
} else {
throw new ExecutionGraphException("The execution attempt " +
- taskExecutionState.getID() + " was not found.");
+ taskExecutionState.getID() + " was not found.");
}
}
@RpcMethod
public SerializedInputSplit requestNextInputSplit(
- final JobVertexID vertexID,
- final ExecutionAttemptID executionAttempt) throws Exception
+ final UUID leaderSessionID,
+ final JobVertexID vertexID,
+ final ExecutionAttemptID executionAttempt) throws Exception
{
+ if (!this.leaderSessionID.equals(leaderSessionID)) {
+ throw new Exception("Leader id not match, expected: " + this.leaderSessionID
+ + ", actual: " + leaderSessionID);
+ }
+
final Execution execution = executionGraph.getRegisteredExecutions().get(executionAttempt);
if (execution == null) {
// can happen when JobManager had already unregistered this execution upon on task failure,
@@ -579,7 +457,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
} catch (Exception ex) {
log.error("Could not serialize the next input split of class {}.", nextInputSplit.getClass(), ex);
IOException reason = new IOException("Could not serialize the next input split of class " +
- nextInputSplit.getClass() + ".", ex);
+ nextInputSplit.getClass() + ".", ex);
vertex.fail(reason);
throw reason;
}
@@ -587,17 +465,31 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
@RpcMethod
public PartitionState requestPartitionState(
- final ResultPartitionID partitionId,
- final ExecutionAttemptID taskExecutionId,
- final IntermediateDataSetID taskResultId)
+ final UUID leaderSessionID,
+ final ResultPartitionID partitionId,
+ final ExecutionAttemptID taskExecutionId,
+ final IntermediateDataSetID taskResultId) throws Exception
{
+ if (!this.leaderSessionID.equals(leaderSessionID)) {
+ throw new Exception("Leader id not match, expected: " + this.leaderSessionID
+ + ", actual: " + leaderSessionID);
+ }
+
final Execution execution = executionGraph.getRegisteredExecutions().get(partitionId.getProducerId());
final ExecutionState state = execution != null ? execution.getState() : null;
return new PartitionState(taskResultId, partitionId.getPartitionId(), state);
}
@RpcMethod
- public Acknowledge scheduleOrUpdateConsumers(final ResultPartitionID partitionID) {
+ public Acknowledge scheduleOrUpdateConsumers(
+ final UUID leaderSessionID,
+ final ResultPartitionID partitionID) throws Exception
+ {
+ if (!this.leaderSessionID.equals(leaderSessionID)) {
+ throw new Exception("Leader id not match, expected: " + this.leaderSessionID
+ + ", actual: " + leaderSessionID);
+ }
+
executionGraph.scheduleOrUpdateConsumers(partitionID);
return Acknowledge.get();
}
@@ -609,223 +501,149 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
@RpcMethod
public void acknowledgeCheckpoint(
- JobID jobID,
- ExecutionAttemptID executionAttemptID,
- long checkpointID,
- CheckpointStateHandles checkpointStateHandles,
- long synchronousDurationMillis,
- long asynchronousDurationMillis,
- long bytesBufferedInAlignment,
- long alignmentDurationNanos) {
- throw new UnsupportedOperationException();
- }
-
- @RpcMethod
- public void declineCheckpoint(
- JobID jobID,
- ExecutionAttemptID executionAttemptID,
- long checkpointID,
- long checkpointTimestamp) {
- throw new UnsupportedOperationException();
- }
-
- @RpcMethod
- public void resourceRemoved(final ResourceID resourceId, final String message) {
- // TODO: remove resource from slot pool
- }
+ final JobID jobID,
+ final ExecutionAttemptID executionAttemptID,
+ final CheckpointMetaData checkpointInfo,
+ final CheckpointStateHandles checkpointState)
+ {
+ final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
+ final AcknowledgeCheckpoint ackMessage =
+ new AcknowledgeCheckpoint(jobID, executionAttemptID, checkpointInfo, checkpointState);
- @RpcMethod
- public void acknowledgeCheckpoint(final AcknowledgeCheckpoint acknowledge) {
- if (executionGraph != null) {
- final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
- if (checkpointCoordinator != null) {
- getRpcService().execute(new Runnable() {
- @Override
- public void run() {
- try {
- if (!checkpointCoordinator.receiveAcknowledgeMessage(acknowledge)) {
- log.info("Received message for non-existing checkpoint {}.",
- acknowledge.getCheckpointId());
- }
- } catch (Exception e) {
- log.error("Error in CheckpointCoordinator while processing {}", acknowledge, e);
+ if (checkpointCoordinator != null) {
+ getRpcService().execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ if (!checkpointCoordinator.receiveAcknowledgeMessage(ackMessage)) {
+ log.info("Received message for non-existing checkpoint {}.",
+ checkpointInfo.getCheckpointId());
}
+ } catch (Exception e) {
+ log.error("Error in CheckpointCoordinator while processing {}", checkpointInfo, e);
}
- });
- }
- else {
- log.error("Received AcknowledgeCheckpoint message for job {} with no CheckpointCoordinator",
- jobGraph.getJobID());
- }
+ }
+ });
} else {
- log.error("Received AcknowledgeCheckpoint for unavailable job {}", jobGraph.getJobID());
+ log.error("Received AcknowledgeCheckpoint message for job {} with no CheckpointCoordinator",
+ jobGraph.getJobID());
}
}
@RpcMethod
- public void declineCheckpoint(final DeclineCheckpoint decline) {
- if (executionGraph != null) {
- final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
- if (checkpointCoordinator != null) {
- getRpcService().execute(new Runnable() {
- @Override
- public void run() {
- try {
- if (!checkpointCoordinator.receiveDeclineMessage(decline)) {
- log.info("Received message for non-existing checkpoint {}.", decline.getCheckpointId());
- }
- } catch (Exception e) {
- log.error("Error in CheckpointCoordinator while processing {}", decline, e);
+ public void declineCheckpoint(
+ final JobID jobID,
+ final ExecutionAttemptID executionAttemptID,
+ final long checkpointID)
+ {
+ final DeclineCheckpoint decline = new DeclineCheckpoint(
+ jobID, executionAttemptID, checkpointID, 0L);
+ final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
+
+ if (checkpointCoordinator != null) {
+ getRpcService().execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ if (!checkpointCoordinator.receiveDeclineMessage(decline)) {
+ log.info("Received message for non-existing checkpoint {}.", decline.getCheckpointId());
}
+ } catch (Exception e) {
+ log.error("Error in CheckpointCoordinator while processing {}", decline, e);
}
- });
- } else {
- log.error("Received DeclineCheckpoint message for job {} with no CheckpointCoordinator",
- jobGraph.getJobID());
- }
+ }
+ });
} else {
- log.error("Received AcknowledgeCheckpoint for unavailable job {}", jobGraph.getJobID());
+ log.error("Received DeclineCheckpoint message for job {} with no CheckpointCoordinator",
+ jobGraph.getJobID());
}
}
@RpcMethod
public KvStateLocation lookupKvStateLocation(final String registrationName) throws Exception {
- if (executionGraph != null) {
- if (log.isDebugEnabled()) {
- log.debug("Lookup key-value state for job {} with registration " +
+ if (log.isDebugEnabled()) {
+ log.debug("Lookup key-value state for job {} with registration " +
"name {}.", jobGraph.getJobID(), registrationName);
- }
+ }
- final KvStateLocationRegistry registry = executionGraph.getKvStateLocationRegistry();
- final KvStateLocation location = registry.getKvStateLocation(registrationName);
- if (location != null) {
- return location;
- } else {
- throw new UnknownKvStateLocation(registrationName);
- }
+ final KvStateLocationRegistry registry = executionGraph.getKvStateLocationRegistry();
+ final KvStateLocation location = registry.getKvStateLocation(registrationName);
+ if (location != null) {
+ return location;
} else {
- throw new IllegalStateException("Received lookup KvState location request for unavailable job " +
- jobGraph.getJobID());
+ throw new UnknownKvStateLocation(registrationName);
}
}
@RpcMethod
public void notifyKvStateRegistered(
- final JobVertexID jobVertexId,
- final KeyGroupRange keyGroupRange,
- final String registrationName,
- final KvStateID kvStateId,
- final KvStateServerAddress kvStateServerAddress)
+ final JobVertexID jobVertexId,
+ final KeyGroupRange keyGroupRange,
+ final String registrationName,
+ final KvStateID kvStateId,
+ final KvStateServerAddress kvStateServerAddress)
{
- if (executionGraph != null) {
- if (log.isDebugEnabled()) {
- log.debug("Key value state registered for job {} under name {}.",
+ if (log.isDebugEnabled()) {
+ log.debug("Key value state registered for job {} under name {}.",
jobGraph.getJobID(), registrationName);
- }
- try {
- executionGraph.getKvStateLocationRegistry().notifyKvStateRegistered(
- jobVertexId, keyGroupRange, registrationName, kvStateId, kvStateServerAddress
- );
- } catch (Exception e) {
- log.error("Failed to notify KvStateRegistry about registration {}.", registrationName);
- }
- } else {
- log.error("Received notify KvState registered request for unavailable job " + jobGraph.getJobID());
+ }
+
+ try {
+ executionGraph.getKvStateLocationRegistry().notifyKvStateRegistered(
+ jobVertexId, keyGroupRange, registrationName, kvStateId, kvStateServerAddress);
+ } catch (Exception e) {
+ log.error("Failed to notify KvStateRegistry about registration {}.", registrationName);
}
}
@RpcMethod
public void notifyKvStateUnregistered(
- JobVertexID jobVertexId,
- KeyGroupRange keyGroupRange,
- String registrationName)
+ JobVertexID jobVertexId,
+ KeyGroupRange keyGroupRange,
+ String registrationName)
{
- if (executionGraph != null) {
- if (log.isDebugEnabled()) {
- log.debug("Key value state unregistered for job {} under name {}.",
+ if (log.isDebugEnabled()) {
+ log.debug("Key value state unregistered for job {} under name {}.",
jobGraph.getJobID(), registrationName);
- }
- try {
- executionGraph.getKvStateLocationRegistry().notifyKvStateUnregistered(
- jobVertexId, keyGroupRange, registrationName
- );
- } catch (Exception e) {
- log.error("Failed to notify KvStateRegistry about registration {}.", registrationName);
- }
- } else {
- log.error("Received notify KvState unregistered request for unavailable job " + jobGraph.getJobID());
}
- }
-
- @RpcMethod
- public Future<TriggerSavepointResponse> triggerSavepoint() throws Exception {
- if (executionGraph != null) {
- final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
- if (checkpointCoordinator != null) {
- try {
- Future<String> savepointFuture = new FlinkFuture<>(
- checkpointCoordinator.triggerSavepoint(System.currentTimeMillis()));
-
- return savepointFuture.handleAsync(new BiFunction<String, Throwable, TriggerSavepointResponse>() {
- @Override
- public TriggerSavepointResponse apply(String savepointPath, Throwable throwable) {
- if (throwable == null) {
- return new TriggerSavepointResponse.Success(jobGraph.getJobID(), savepointPath);
- }
- else {
- return new TriggerSavepointResponse.Failure(jobGraph.getJobID(),
- new Exception("Failed to complete savepoint", throwable));
- }
- }
- }, getMainThreadExecutor());
- } catch (Exception e) {
- FlinkCompletableFuture<TriggerSavepointResponse> future = new FlinkCompletableFuture<>();
- future.complete(new TriggerSavepointResponse.Failure(jobGraph.getJobID(),
- new Exception("Failed to trigger savepoint", e)));
- return future;
- }
- } else {
- FlinkCompletableFuture<TriggerSavepointResponse> future = new FlinkCompletableFuture<>();
- future.complete(new TriggerSavepointResponse.Failure(jobGraph.getJobID(),
- new IllegalStateException("Checkpointing disabled. You can enable it via the execution " +
- "environment of your job.")));
- return future;
- }
- } else {
- FlinkCompletableFuture<TriggerSavepointResponse> future = new FlinkCompletableFuture<>();
- future.complete(new TriggerSavepointResponse.Failure(jobGraph.getJobID(),
- new IllegalArgumentException("Received trigger savepoint request for unavailable job " +
- jobGraph.getJobID())));
- return future;
+ try {
+ executionGraph.getKvStateLocationRegistry().notifyKvStateUnregistered(
+ jobVertexId, keyGroupRange, registrationName);
+ } catch (Exception e) {
+ log.error("Failed to notify KvStateRegistry about registration {}.", registrationName);
}
}
@RpcMethod
- public DisposeSavepointResponse disposeSavepoint(final String savepointPath) {
- try {
- log.info("Disposing savepoint at {}.", savepointPath);
+ public Future<String> triggerSavepoint(final UUID leaderSessionID, final String targetDirectory) throws Exception {
+ if (!this.leaderSessionID.equals(leaderSessionID)) {
+ throw new Exception("Leader id not match, expected: " + this.leaderSessionID
+ + ", actual: " + leaderSessionID);
+ }
- // check whether the savepoint exists
- savepointStore.loadSavepoint(savepointPath);
+ final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
+ if (checkpointCoordinator != null) {
+ Future<CompletedCheckpoint> completedCheckpointFuture =
+ checkpointCoordinator.triggerSavepoint(System.currentTimeMillis(), targetDirectory);
- savepointStore.disposeSavepoint(savepointPath);
- return new DisposeSavepointResponse.Success();
- } catch (Exception e) {
- log.error("Failed to dispose savepoint at {}.", savepointPath, e);
- return new DisposeSavepointResponse.Failure(e);
+ return completedCheckpointFuture.thenApplyAsync(new ApplyFunction<CompletedCheckpoint, String>() {
+ @Override
+ public String apply(CompletedCheckpoint checkpoint) {
+ return checkpoint.getExternalPath();
+ }
+ }, executionContext);
+ } else {
+ throw new IllegalStateException("Checkpointing disabled. You can enable it via the execution " +
+ "environment of your job.");
}
}
@RpcMethod
public ClassloadingProps requestClassloadingProps() throws Exception {
- if (executionGraph != null) {
- return new ClassloadingProps(libraryCacheManager.getBlobServerPort(),
+ return new ClassloadingProps(libraryCacheManager.getBlobServerPort(),
executionGraph.getRequiredJarFiles(),
executionGraph.getRequiredClasspaths());
- } else {
- throw new Exception("Received classloading props request for unavailable job " + jobGraph.getJobID());
- }
}
//----------------------------------------------------------------------------------------------
@@ -838,12 +656,11 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
public void run() {
log.error("Fatal error occurred on JobManager, cause: {}", cause.getMessage(), cause);
shutDown();
- jobCompletionActions.onFatalError(cause);
+ errorHandler.onFatalError(cause);
}
});
}
- // TODO - wrap this as StatusListenerMessenger's callback with rpc main thread
private void jobStatusChanged(final JobStatus newJobStatus, long timestamp, final Throwable error) {
final JobID jobID = executionGraph.getJobID();
final String jobName = executionGraph.getJobName();
@@ -871,36 +688,33 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
if (newJobStatus == JobStatus.FINISHED) {
try {
final Map<String, SerializedValue<Object>> accumulatorResults =
- executionGraph.getAccumulatorsSerialized();
+ executionGraph.getAccumulatorsSerialized();
final SerializedJobExecutionResult result = new SerializedJobExecutionResult(
- jobID, 0, accumulatorResults // TODO get correct job duration
+ jobID, 0, accumulatorResults // TODO get correct job duration
);
jobCompletionActions.jobFinished(result.toJobExecutionResult(userCodeLoader));
} catch (Exception e) {
log.error("Cannot fetch final accumulators for job {} ({})", jobName, jobID, e);
final JobExecutionException exception = new JobExecutionException(
- jobID, "Failed to retrieve accumulator results.", e);
+ jobID, "Failed to retrieve accumulator results.", e);
// TODO should we also notify client?
jobCompletionActions.jobFailed(exception);
}
- }
- else if (newJobStatus == JobStatus.CANCELED) {
+ } else if (newJobStatus == JobStatus.CANCELED) {
final Throwable unpackedError = SerializedThrowable.get(error, userCodeLoader);
final JobExecutionException exception = new JobExecutionException(
- jobID, "Job was cancelled.", unpackedError);
+ jobID, "Job was cancelled.", unpackedError);
// TODO should we also notify client?
jobCompletionActions.jobFailed(exception);
- }
- else if (newJobStatus == JobStatus.FAILED) {
+ } else if (newJobStatus == JobStatus.FAILED) {
final Throwable unpackedError = SerializedThrowable.get(error, userCodeLoader);
final JobExecutionException exception = new JobExecutionException(
- jobID, "Job execution failed.", unpackedError);
+ jobID, "Job execution failed.", unpackedError);
// TODO should we also notify client?
jobCompletionActions.jobFailed(exception);
- }
- else {
+ } else {
final JobExecutionException exception = new JobExecutionException(
- jobID, newJobStatus + " is not a terminal state.");
+ jobID, newJobStatus + " is not a terminal state.");
// TODO should we also notify client?
jobCompletionActions.jobFailed(exception);
throw new RuntimeException(exception);
@@ -909,7 +723,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
}
private void notifyOfNewResourceManagerLeader(
- final String resourceManagerAddress, final UUID resourceManagerLeaderId)
+ final String resourceManagerAddress, final UUID resourceManagerLeaderId)
{
// IMPORTANT: executed by main thread to avoid concurrence
runAsync(new Runnable() {
@@ -918,17 +732,15 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
if (resourceManagerConnection != null) {
if (resourceManagerAddress != null) {
if (resourceManagerAddress.equals(resourceManagerConnection.getTargetAddress())
- && resourceManagerLeaderId.equals(resourceManagerConnection.getTargetLeaderId()))
- {
+ && resourceManagerLeaderId.equals(resourceManagerConnection.getTargetLeaderId())) {
// both address and leader id are not changed, we can keep the old connection
return;
}
log.info("ResourceManager leader changed from {} to {}. Registering at new leader.",
- resourceManagerConnection.getTargetAddress(), resourceManagerAddress);
- }
- else {
+ resourceManagerConnection.getTargetAddress(), resourceManagerAddress);
+ } else {
log.info("Current ResourceManager {} lost leader status. Waiting for new ResourceManager leader.",
- resourceManagerConnection.getTargetAddress());
+ resourceManagerConnection.getTargetAddress());
}
}
@@ -937,8 +749,8 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
if (resourceManagerAddress != null) {
log.info("Attempting to register at ResourceManager {}", resourceManagerAddress);
resourceManagerConnection = new ResourceManagerConnection(
- log, jobGraph.getJobID(), leaderSessionID,
- resourceManagerAddress, resourceManagerLeaderId, executionContext);
+ log, jobGraph.getJobID(), leaderSessionID,
+ resourceManagerAddress, resourceManagerLeaderId, executionContext);
resourceManagerConnection.start();
}
}
@@ -952,26 +764,14 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
// TODO - add tests for comment in https://github.com/apache/flink/pull/2565
// verify the response with current connection
if (resourceManagerConnection != null
- && resourceManagerConnection.getTargetLeaderId().equals(success.getResourceManagerLeaderId())) {
+ && resourceManagerConnection.getTargetLeaderId().equals(success.getResourceManagerLeaderId())) {
log.info("JobManager successfully registered at ResourceManager, leader id: {}.",
- success.getResourceManagerLeaderId());
+ success.getResourceManagerLeaderId());
}
}
});
}
- private void disposeCommunicationWithResourceManager() {
- // 1. stop the leader retriever so we will not receiving updates anymore
- try {
- resourceManagerLeaderRetriever.stop();
- } catch (Exception e) {
- log.warn("Failed to stop resource manager leader retriever.");
- }
-
- // 2. close current connection with ResourceManager if exists
- closeResourceManagerConnection();
- }
-
private void closeResourceManagerConnection() {
if (resourceManagerConnection != null) {
resourceManagerConnection.close();
@@ -980,34 +780,6 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
}
//----------------------------------------------------------------------------------------------
- // Helper methods
- //----------------------------------------------------------------------------------------------
-
- /**
- * Converts JobVertexIDs to corresponding ExecutionJobVertexes
- *
- * @param executionGraph The execution graph that holds the relationship
- * @param vertexIDs The vertexIDs need to be converted
- * @return The corresponding ExecutionJobVertexes
- * @throws JobExecutionException
- */
- private static List<ExecutionJobVertex> getExecutionJobVertexWithId(
- final ExecutionGraph executionGraph, final List<JobVertexID> vertexIDs)
- throws JobExecutionException
- {
- final List<ExecutionJobVertex> ret = new ArrayList<>(vertexIDs.size());
- for (JobVertexID vertexID : vertexIDs) {
- final ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(vertexID);
- if (executionJobVertex == null) {
- throw new JobExecutionException(executionGraph.getJobID(),
- "The snapshot checkpointing settings refer to non-existent vertex " + vertexID);
- }
- ret.add(executionJobVertex);
- }
- return ret;
- }
-
- //----------------------------------------------------------------------------------------------
// Utility classes
//----------------------------------------------------------------------------------------------
@@ -1024,19 +796,19 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
}
private class ResourceManagerConnection
- extends RegisteredRpcConnection<ResourceManagerGateway, JobMasterRegistrationSuccess>
+ extends RegisteredRpcConnection<ResourceManagerGateway, JobMasterRegistrationSuccess>
{
private final JobID jobID;
private final UUID jobManagerLeaderID;
ResourceManagerConnection(
- final Logger log,
- final JobID jobID,
- final UUID jobManagerLeaderID,
- final String resourceManagerAddress,
- final UUID resourceManagerLeaderID,
- final Executor executor)
+ final Logger log,
+ final JobID jobID,
+ final UUID jobManagerLeaderID,
+ final String resourceManagerAddress,
+ final UUID resourceManagerLeaderID,
+ final Executor executor)
{
super(log, resourceManagerAddress, resourceManagerLeaderID, executor);
this.jobID = checkNotNull(jobID);
@@ -1046,12 +818,12 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
@Override
protected RetryingRegistration<ResourceManagerGateway, JobMasterRegistrationSuccess> generateRegistration() {
return new RetryingRegistration<ResourceManagerGateway, JobMasterRegistrationSuccess>(
- log, getRpcService(), "ResourceManager", ResourceManagerGateway.class,
- getTargetAddress(), getTargetLeaderId())
+ log, getRpcService(), "ResourceManager", ResourceManagerGateway.class,
+ getTargetAddress(), getTargetLeaderId())
{
@Override
protected Future<RegistrationResponse> invokeRegistration(ResourceManagerGateway gateway, UUID leaderId,
- long timeoutMillis) throws Exception
+ long timeoutMillis) throws Exception
{
Time timeout = Time.milliseconds(timeoutMillis);
return gateway.registerJobMaster(leaderId, jobManagerLeaderID, getAddress(), jobID, timeout);
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
index 4b51258..b27b41c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
@@ -18,6 +18,8 @@
package org.apache.flink.runtime.jobmaster;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.checkpoint.CheckpointCoordinatorGateway;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
@@ -29,15 +31,11 @@ import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobmaster.message.ClassloadingProps;
-import org.apache.flink.runtime.jobmaster.message.DisposeSavepointResponse;
-import org.apache.flink.runtime.jobmaster.message.NextInputSplit;
-import org.apache.flink.runtime.jobmaster.message.TriggerSavepointResponse;
-import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
-import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint;
+import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.query.KvStateID;
import org.apache.flink.runtime.query.KvStateLocation;
import org.apache.flink.runtime.query.KvStateServerAddress;
-import org.apache.flink.runtime.rpc.RpcGateway;
+import org.apache.flink.runtime.rpc.RpcTimeout;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.KvState;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
@@ -49,52 +47,56 @@ import java.util.UUID;
*/
public interface JobMasterGateway extends CheckpointCoordinatorGateway {
- /**
- * Starting the job under the given leader session ID.
- */
- void startJob(final UUID leaderSessionID);
+ // ------------------------------------------------------------------------
+ // Job start and stop methods
+ // ------------------------------------------------------------------------
- /**
- * Suspending job, all the running tasks will be cancelled, and runtime status will be cleared.
- * Should re-submit the job before restarting it.
- *
- * @param cause The reason of why this job been suspended.
- */
- void suspendJob(final Throwable cause);
+ void startJobExecution();
+
+ void suspendExecution(Throwable cause);
+
+ // ------------------------------------------------------------------------
/**
* Updates the task execution state for a given task.
*
+ * @param leaderSessionID The leader id of JobManager
* @param taskExecutionState New task execution state for a given task
* @return Future flag of the task execution state update result
*/
- Future<Acknowledge> updateTaskExecutionState(TaskExecutionState taskExecutionState);
+ Future<Acknowledge> updateTaskExecutionState(
+ final UUID leaderSessionID,
+ final TaskExecutionState taskExecutionState);
/**
* Requesting next input split for the {@link ExecutionJobVertex}. The next input split is sent back to the sender
* as a {@link SerializedInputSplit} message.
*
+ * @param leaderSessionID The leader id of JobManager
* @param vertexID The job vertex id
* @param executionAttempt The execution attempt id
* @return The future of the input split. If there is no further input split, will return an empty object.
*/
Future<SerializedInputSplit> requestNextInputSplit(
- final JobVertexID vertexID,
- final ExecutionAttemptID executionAttempt);
+ final UUID leaderSessionID,
+ final JobVertexID vertexID,
+ final ExecutionAttemptID executionAttempt);
/**
* Requests the current state of the partition.
* The state of a partition is currently bound to the state of the producing execution.
*
+ * @param leaderSessionID The leader id of JobManager
* @param partitionId The partition ID of the partition to request the state of.
* @param taskExecutionId The execution attempt ID of the task requesting the partition state.
* @param taskResultId The input gate ID of the task requesting the partition state.
* @return The future of the partition state
*/
Future<PartitionState> requestPartitionState(
- final ResultPartitionID partitionId,
- final ExecutionAttemptID taskExecutionId,
- final IntermediateDataSetID taskResultId);
+ final UUID leaderSessionID,
+ final ResultPartitionID partitionId,
+ final ExecutionAttemptID taskExecutionId,
+ final IntermediateDataSetID taskResultId);
/**
* Notifies the JobManager about available data for a produced partition.
@@ -105,11 +107,15 @@ public interface JobMasterGateway extends CheckpointCoordinatorGateway {
* <p>
* The JobManager then can decide when to schedule the partition consumers of the given session.
*
- * @param partitionID The partition which has already produced data
- * @param timeout before the rpc call fails
+ * @param leaderSessionID The leader id of JobManager
+ * @param partitionID The partition which has already produced data
+ * @param timeout before the rpc call fails
* @return Future acknowledge of the schedule or update operation
*/
- Future<Acknowledge> scheduleOrUpdateConsumers(final ResultPartitionID partitionID, @RpcTimeout final Time timeout);
+ Future<Acknowledge> scheduleOrUpdateConsumers(
+ final UUID leaderSessionID,
+ final ResultPartitionID partitionID,
+ @RpcTimeout final Time timeout);
/**
* Disconnects the given {@link org.apache.flink.runtime.taskexecutor.TaskExecutor} from the
@@ -118,31 +124,6 @@ public interface JobMasterGateway extends CheckpointCoordinatorGateway {
* @param resourceID identifying the TaskManager to disconnect
*/
void disconnectTaskManager(ResourceID resourceID);
- void scheduleOrUpdateConsumers(final ResultPartitionID partitionID);
-
- /**
- * Notifies the JobManager about the removal of a resource.
- *
- * @param resourceId The ID under which the resource is registered.
- * @param message Optional message with details, for logging and debugging.
- */
-
- void resourceRemoved(final ResourceID resourceId, final String message);
-
- /**
- * Notifies the JobManager that the checkpoint of an individual task is completed.
- *
- * @param acknowledge The acknowledge message of the checkpoint
- */
- void acknowledgeCheckpoint(final AcknowledgeCheckpoint acknowledge);
-
- /**
- * Notifies the JobManager that a checkpoint request could not be heeded.
- * This can happen if a Task is already in RUNNING state but is internally not yet ready to perform checkpoints.
- *
- * @param decline The decline message of the checkpoint
- */
- void declineCheckpoint(final DeclineCheckpoint decline);
/**
* Requests a {@link KvStateLocation} for the specified {@link KvState} registration name.
@@ -150,7 +131,7 @@ public interface JobMasterGateway extends CheckpointCoordinatorGateway {
* @param registrationName Name under which the KvState has been registered.
* @return Future of the requested {@link KvState} location
*/
- Future<KvStateLocation> lookupKvStateLocation(final String registrationName) throws Exception;
+ Future<KvStateLocation> lookupKvStateLocation(final String registrationName);
/**
* @param jobVertexId JobVertexID the KvState instance belongs to.
@@ -160,11 +141,11 @@ public interface JobMasterGateway extends CheckpointCoordinatorGateway {
* @param kvStateServerAddress Server address where to find the KvState instance.
*/
void notifyKvStateRegistered(
- final JobVertexID jobVertexId,
- final KeyGroupRange keyGroupRange,
- final String registrationName,
- final KvStateID kvStateId,
- final KvStateServerAddress kvStateServerAddress);
+ final JobVertexID jobVertexId,
+ final KeyGroupRange keyGroupRange,
+ final String registrationName,
+ final KvStateID kvStateId,
+ final KvStateServerAddress kvStateServerAddress);
/**
* @param jobVertexId JobVertexID the KvState instance belongs to.
@@ -172,24 +153,18 @@ public interface JobMasterGateway extends CheckpointCoordinatorGateway {
* @param registrationName Name under which the KvState has been registered.
*/
void notifyKvStateUnregistered(
- JobVertexID jobVertexId,
- KeyGroupRange keyGroupRange,
- String registrationName);
+ JobVertexID jobVertexId,
+ KeyGroupRange keyGroupRange,
+ String registrationName);
/**
* Notifies the JobManager to trigger a savepoint for this job.
*
- * @return Future of the savepoint trigger response.
- */
- Future<TriggerSavepointResponse> triggerSavepoint();
-
- /**
- * Notifies the Jobmanager to dispose specified savepoint.
- *
- * @param savepointPath The path of the savepoint.
- * @return The future of the savepoint disponse response.
+ * @param leaderSessionID The leader id of JobManager
+ * @param targetDirectory The directory where the savepoint should be stored
+ * @return The savepoint path
*/
- Future<DisposeSavepointResponse> disposeSavepoint(final String savepointPath);
+ Future<String> triggerSavepoint(UUID leaderSessionID, String targetDirectory);
/**
* Request the classloading props of this job.
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java
index e8fb5bb..019ccfe 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java
@@ -25,6 +25,8 @@ import org.apache.flink.runtime.client.JobExecutionException;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobmanager.OnCompletionActions;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcService;
import org.slf4j.Logger;
@@ -62,6 +64,9 @@ public class MiniClusterJobDispatcher {
/** al the services that the JobManager needs, such as BLOB service, factories, etc */
private final JobManagerServices jobManagerServices;
+ /** Registry for all metrics in the mini cluster */
+ private final MetricRegistry metricRegistry;
+
/** The number of JobManagers to launch (more than one simulates a high-availability setup) */
private final int numJobManagers;
@@ -86,8 +91,9 @@ public class MiniClusterJobDispatcher {
public MiniClusterJobDispatcher(
Configuration config,
RpcService rpcService,
- HighAvailabilityServices haServices) throws Exception {
- this(config, rpcService, haServices, 1);
+ HighAvailabilityServices haServices,
+ MetricRegistry metricRegistry) throws Exception {
+ this(config, rpcService, haServices, metricRegistry, 1);
}
/**
@@ -106,16 +112,18 @@ public class MiniClusterJobDispatcher {
Configuration config,
RpcService rpcService,
HighAvailabilityServices haServices,
+ MetricRegistry metricRegistry,
int numJobManagers) throws Exception {
checkArgument(numJobManagers >= 1);
this.configuration = checkNotNull(config);
this.rpcService = checkNotNull(rpcService);
this.haServices = checkNotNull(haServices);
+ this.metricRegistry = checkNotNull(metricRegistry);
this.numJobManagers = numJobManagers;
LOG.info("Creating JobMaster services");
- this.jobManagerServices = JobManagerServices.fromConfiguration(config);
+ this.jobManagerServices = JobManagerServices.fromConfiguration(config, haServices);
}
// ------------------------------------------------------------------------
@@ -140,9 +148,8 @@ public class MiniClusterJobDispatcher {
if (runners != null) {
this.runners = null;
- Exception shutdownException = new Exception("The MiniCluster is shutting down");
for (JobManagerRunner runner : runners) {
- runner.shutdown(shutdownException);
+ runner.shutdown();
}
}
}
@@ -171,9 +178,9 @@ public class MiniClusterJobDispatcher {
checkState(!shutdown, "mini cluster is shut down");
checkState(runners == null, "mini cluster can only execute one job at a time");
- OnCompletionActions onJobCompletion = new DetachedFinalizer(numJobManagers);
+ DetachedFinalizer finalizer = new DetachedFinalizer(numJobManagers);
- this.runners = startJobRunners(job, onJobCompletion);
+ this.runners = startJobRunners(job, finalizer, finalizer);
}
}
@@ -191,17 +198,17 @@ public class MiniClusterJobDispatcher {
checkNotNull(job);
LOG.info("Received job for blocking execution {} ({})", job.getName(), job.getJobID());
- final BlockingJobSync onJobCompletion = new BlockingJobSync(job.getJobID(), numJobManagers);
+ final BlockingJobSync sync = new BlockingJobSync(job.getJobID(), numJobManagers);
synchronized (lock) {
checkState(!shutdown, "mini cluster is shut down");
checkState(runners == null, "mini cluster can only execute one job at a time");
- this.runners = startJobRunners(job, onJobCompletion);
+ this.runners = startJobRunners(job, sync, sync);
}
try {
- return onJobCompletion.getResult();
+ return sync.getResult();
}
finally {
// always clear the status for the next job
@@ -209,24 +216,26 @@ public class MiniClusterJobDispatcher {
}
}
- private JobManagerRunner[] startJobRunners(JobGraph job, OnCompletionActions onCompletion) throws JobExecutionException {
+ private JobManagerRunner[] startJobRunners(
+ JobGraph job,
+ OnCompletionActions onCompletion,
+ FatalErrorHandler errorHandler) throws JobExecutionException {
LOG.info("Starting {} JobMaster(s) for job {} ({})", numJobManagers, job.getName(), job.getJobID());
JobManagerRunner[] runners = new JobManagerRunner[numJobManagers];
for (int i = 0; i < numJobManagers; i++) {
try {
runners[i] = new JobManagerRunner(job, configuration,
- rpcService, haServices, jobManagerServices, onCompletion);
+ rpcService, haServices, jobManagerServices, metricRegistry,
+ onCompletion, errorHandler);
runners[i].start();
}
catch (Throwable t) {
// shut down all the ones so far
- Exception shutdownCause = new Exception("Failed to properly start all mini cluster JobManagers", t);
-
for (int k = 0; k <= i; k++) {
try {
if (runners[i] != null) {
- runners[i].shutdown(shutdownCause);
+ runners[i].shutdown();
}
} catch (Throwable ignored) {
// silent shutdown
@@ -244,15 +253,15 @@ public class MiniClusterJobDispatcher {
// test methods to simulate job master failures
// ------------------------------------------------------------------------
- public void killJobMaster(int which) {
- checkArgument(which >= 0 && which < numJobManagers, "no such job master");
- checkState(!shutdown, "mini cluster is shut down");
-
- JobManagerRunner[] runners = this.runners;
- checkState(runners != null, "mini cluster it not executing a job right now");
-
- runners[which].shutdown(new Throwable("kill JobManager"));
- }
+// public void killJobMaster(int which) {
+// checkArgument(which >= 0 && which < numJobManagers, "no such job master");
+// checkState(!shutdown, "mini cluster is shut down");
+//
+// JobManagerRunner[] runners = this.runners;
+// checkState(runners != null, "mini cluster it not executing a job right now");
+//
+// runners[which].shutdown(new Throwable("kill JobManager"));
+// }
// ------------------------------------------------------------------------
// utility classes
@@ -263,7 +272,7 @@ public class MiniClusterJobDispatcher {
* In the case of a high-availability test setup, there may be multiple runners.
* After that, it marks the mini cluster as ready to receive new jobs.
*/
- private class DetachedFinalizer implements OnCompletionActions {
+ private class DetachedFinalizer implements OnCompletionActions, FatalErrorHandler {
private final AtomicInteger numJobManagersToWaitFor;
@@ -308,7 +317,7 @@ public class MiniClusterJobDispatcher {
* That way it is guaranteed that after the blocking job submit call returns,
* the dispatcher is immediately free to accept another job.
*/
- private static class BlockingJobSync implements OnCompletionActions {
+ private static class BlockingJobSync implements OnCompletionActions, FatalErrorHandler {
private final JobID jobId;
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/DisposeSavepointResponse.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/DisposeSavepointResponse.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/DisposeSavepointResponse.java
deleted file mode 100644
index 42bfc71..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/DisposeSavepointResponse.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.jobmaster.message;
-
-import java.io.Serializable;
-
-/**
- * The response of the dispose savepoint request to JobManager.
- */
-public abstract class DisposeSavepointResponse implements Serializable {
-
- private static final long serialVersionUID = 6008792963949369567L;
-
- public static class Success extends DisposeSavepointResponse implements Serializable {
-
- private static final long serialVersionUID = 1572462960008711415L;
- }
-
- public static class Failure extends DisposeSavepointResponse implements Serializable {
-
- private static final long serialVersionUID = -7505308325483022458L;
-
- private final Throwable cause;
-
- public Failure(final Throwable cause) {
- this.cause = cause;
- }
-
- public Throwable getCause() {
- return cause;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/TriggerSavepointResponse.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/TriggerSavepointResponse.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/TriggerSavepointResponse.java
deleted file mode 100644
index 0b0edc5..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/TriggerSavepointResponse.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.jobmaster.message;
-
-import org.apache.flink.api.common.JobID;
-
-import java.io.Serializable;
-
-/**
- * The response of the trigger savepoint request to JobManager.
- */
-public abstract class TriggerSavepointResponse implements Serializable {
-
- private static final long serialVersionUID = 3139327824611807707L;
-
- private final JobID jobID;
-
- public JobID getJobID() {
- return jobID;
- }
-
- public TriggerSavepointResponse(final JobID jobID) {
- this.jobID = jobID;
- }
-
- public static class Success extends TriggerSavepointResponse implements Serializable {
-
- private static final long serialVersionUID = -1100637460388881776L;
-
- private final String savepointPath;
-
- public Success(final JobID jobID, final String savepointPath) {
- super(jobID);
- this.savepointPath = savepointPath;
- }
-
- public String getSavepointPath() {
- return savepointPath;
- }
- }
-
- public static class Failure extends TriggerSavepointResponse implements Serializable {
-
- private static final long serialVersionUID = -1668479003490615139L;
-
- private final Throwable cause;
-
- public Failure(final JobID jobID, final Throwable cause) {
- super(jobID);
- this.cause = cause;
- }
-
- public Throwable getCause() {
- return cause;
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java
index 2052f98..4b9100a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java
@@ -33,8 +33,8 @@ import java.util.concurrent.TimeUnit;
public interface RpcService {
/**
- * Return the address under which the rpc service can be reached. If the rpc service cannot be
- * contacted remotely, then it will return an empty string.
+ * Return the hostname or host address under which the rpc service can be reached.
+ * If the rpc service cannot be contacted remotely, then it will return an empty string.
*
* @return Address of the rpc service or empty string if local rpc service
*/
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerConnection.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerConnection.java
index ef62ef1..6fcd082 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerConnection.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobManagerConnection.java
@@ -26,11 +26,16 @@ import org.apache.flink.runtime.taskmanager.CheckpointResponder;
import org.apache.flink.runtime.taskmanager.TaskManagerActions;
import org.apache.flink.util.Preconditions;
+import java.util.UUID;
+
/**
* Container class for JobManager specific communication utils used by the {@link TaskExecutor}.
*/
public class JobManagerConnection {
+ // Job master leader session id
+ private final UUID jobMasterLeaderId;
+
// Gateway to the job master
private final JobMasterGateway jobMasterGateway;
@@ -50,13 +55,15 @@ public class JobManagerConnection {
private final PartitionStateChecker partitionStateChecker;
public JobManagerConnection(
- JobMasterGateway jobMasterGateway,
- TaskManagerActions taskManagerActions,
- CheckpointResponder checkpointResponder,
- LibraryCacheManager libraryCacheManager,
- ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
- PartitionStateChecker partitionStateChecker) {
-
+ UUID jobMasterLeaderId,
+ JobMasterGateway jobMasterGateway,
+ TaskManagerActions taskManagerActions,
+ CheckpointResponder checkpointResponder,
+ LibraryCacheManager libraryCacheManager,
+ ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
+ PartitionStateChecker partitionStateChecker)
+ {
+ this.jobMasterLeaderId = Preconditions.checkNotNull(jobMasterLeaderId);
this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
this.taskManagerActions = Preconditions.checkNotNull(taskManagerActions);
this.checkpointResponder = Preconditions.checkNotNull(checkpointResponder);
@@ -65,6 +72,10 @@ public class JobManagerConnection {
this.partitionStateChecker = Preconditions.checkNotNull(partitionStateChecker);
}
+ public UUID getJobMasterLeaderId() {
+ return jobMasterLeaderId;
+ }
+
public JobMasterGateway getJobManagerGateway() {
return jobMasterGateway;
}
[21/50] [abbrv] flink git commit: [FLINK-4694] [rpc] Add termination
futures to RpcEndpoint and RpcService
Posted by tr...@apache.org.
[FLINK-4694] [rpc] Add termination futures to RpcEndpoint and RpcService
The termination futures can be used to wait for the termination of the respective component.
This closes #2558.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1a9e6447
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1a9e6447
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1a9e6447
Branch: refs/heads/flip-6
Commit: 1a9e6447dbb63d138f97b55c15d1158b66e32286
Parents: 8adceed
Author: Till Rohrmann <tr...@apache.org>
Authored: Tue Sep 27 18:17:42 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:30 2016 +0100
----------------------------------------------------------------------
.../concurrent/impl/FlinkCompletableFuture.java | 9 ++---
.../apache/flink/runtime/rpc/RpcEndpoint.java | 9 +++++
.../apache/flink/runtime/rpc/RpcService.java | 7 ++++
.../apache/flink/runtime/rpc/SelfGateway.java | 34 ++++++++++++++++++
.../runtime/rpc/akka/AkkaInvocationHandler.java | 22 ++++++++++--
.../flink/runtime/rpc/akka/AkkaRpcActor.java | 17 ++++++++-
.../flink/runtime/rpc/akka/AkkaRpcService.java | 32 +++++++++++++++--
.../runtime/rpc/TestingSerialRpcService.java | 10 +++++-
.../runtime/rpc/akka/AkkaRpcActorTest.java | 36 ++++++++++++++++++++
.../runtime/rpc/akka/AkkaRpcServiceTest.java | 29 ++++++++++++++++
10 files changed, 193 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1a9e6447/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/impl/FlinkCompletableFuture.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/impl/FlinkCompletableFuture.java b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/impl/FlinkCompletableFuture.java
index c8b86ed..14686d7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/impl/FlinkCompletableFuture.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/impl/FlinkCompletableFuture.java
@@ -20,7 +20,6 @@ package org.apache.flink.runtime.concurrent.impl;
import akka.dispatch.Futures;
import org.apache.flink.runtime.concurrent.CompletableFuture;
-import org.apache.flink.util.Preconditions;
import scala.concurrent.Promise;
import scala.concurrent.Promise$;
@@ -63,10 +62,12 @@ public class FlinkCompletableFuture<T> extends FlinkFuture<T> implements Complet
@Override
public boolean completeExceptionally(Throwable t) {
- Preconditions.checkNotNull(t);
-
try {
- promise.failure(t);
+ if (t == null) {
+ promise.failure(new NullPointerException("Throwable was null."));
+ } else {
+ promise.failure(t);
+ }
return true;
} catch (IllegalStateException e) {
http://git-wip-us.apache.org/repos/asf/flink/blob/1a9e6447/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
index 79961f7..f93a2e2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
@@ -173,6 +173,15 @@ public abstract class RpcEndpoint<C extends RpcGateway> {
return rpcService;
}
+ /**
+ * Return a future which is completed when the rpc endpoint has been terminated.
+ *
+ * @return Future which is completed when the rpc endpoint has been terminated.
+ */
+ public Future<Void> getTerminationFuture() {
+ return ((SelfGateway)self).getTerminationFuture();
+ }
+
// ------------------------------------------------------------------------
// Asynchronous executions
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1a9e6447/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java
index 96844ed..2052f98 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcService.java
@@ -77,6 +77,13 @@ public interface RpcService {
void stopService();
/**
+ * Returns a future indicating when the RPC service has been shut down.
+ *
+ * @return Termination future
+ */
+ Future<Void> getTerminationFuture();
+
+ /**
* Gets the executor, provided by this RPC service. This executor can be used for example for
* the {@code handleAsync(...)} or {@code thenAcceptAsync(...)} methods of futures.
*
http://git-wip-us.apache.org/repos/asf/flink/blob/1a9e6447/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/SelfGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/SelfGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/SelfGateway.java
new file mode 100644
index 0000000..ed8ef9d
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/SelfGateway.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rpc;
+
+import org.apache.flink.runtime.concurrent.Future;
+
+/**
+ * Interface for self gateways
+ */
+public interface SelfGateway {
+
+ /**
+ * Return a future which is completed when the rpc endpoint has been terminated.
+ *
+ * @return Future indicating when the rpc endpoint has been terminated
+ */
+ Future<Void> getTerminationFuture();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1a9e6447/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaInvocationHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaInvocationHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaInvocationHandler.java
index 8f4deff..709ff92 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaInvocationHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaInvocationHandler.java
@@ -25,6 +25,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.concurrent.impl.FlinkFuture;
import org.apache.flink.runtime.rpc.MainThreadExecutable;
+import org.apache.flink.runtime.rpc.SelfGateway;
import org.apache.flink.runtime.rpc.RpcGateway;
import org.apache.flink.runtime.rpc.RpcTimeout;
import org.apache.flink.runtime.rpc.StartStoppable;
@@ -52,7 +53,7 @@ import static org.apache.flink.util.Preconditions.checkArgument;
* rpc in a {@link LocalRpcInvocation} message and then sends it to the {@link AkkaRpcActor} where it is
* executed.
*/
-class AkkaInvocationHandler implements InvocationHandler, AkkaGateway, MainThreadExecutable, StartStoppable {
+class AkkaInvocationHandler implements InvocationHandler, AkkaGateway, MainThreadExecutable, StartStoppable, SelfGateway {
private static final Logger LOG = Logger.getLogger(AkkaInvocationHandler.class);
private final String address;
@@ -67,12 +68,22 @@ class AkkaInvocationHandler implements InvocationHandler, AkkaGateway, MainThrea
private final long maximumFramesize;
- AkkaInvocationHandler(String address, ActorRef rpcEndpoint, Time timeout, long maximumFramesize) {
+ // null if gateway; otherwise non-null
+ private final Future<Void> terminationFuture;
+
+ AkkaInvocationHandler(
+ String address,
+ ActorRef rpcEndpoint,
+ Time timeout,
+ long maximumFramesize,
+ Future<Void> terminationFuture) {
+
this.address = Preconditions.checkNotNull(address);
this.rpcEndpoint = Preconditions.checkNotNull(rpcEndpoint);
this.isLocal = this.rpcEndpoint.path().address().hasLocalScope();
this.timeout = Preconditions.checkNotNull(timeout);
this.maximumFramesize = maximumFramesize;
+ this.terminationFuture = terminationFuture;
}
@Override
@@ -83,7 +94,7 @@ class AkkaInvocationHandler implements InvocationHandler, AkkaGateway, MainThrea
if (declaringClass.equals(AkkaGateway.class) || declaringClass.equals(MainThreadExecutable.class) ||
declaringClass.equals(Object.class) || declaringClass.equals(StartStoppable.class) ||
- declaringClass.equals(RpcGateway.class)) {
+ declaringClass.equals(RpcGateway.class) || declaringClass.equals(SelfGateway.class)) {
result = method.invoke(this, args);
} else {
String methodName = method.getName();
@@ -300,4 +311,9 @@ class AkkaInvocationHandler implements InvocationHandler, AkkaGateway, MainThrea
public String getAddress() {
return address;
}
+
+ @Override
+ public Future<Void> getTerminationFuture() {
+ return terminationFuture;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1a9e6447/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java
index 1b456a7..c21383a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java
@@ -24,6 +24,7 @@ import akka.actor.UntypedActorWithStash;
import akka.dispatch.Futures;
import akka.japi.Procedure;
import akka.pattern.Patterns;
+import org.apache.flink.runtime.concurrent.CompletableFuture;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.concurrent.impl.FlinkFuture;
import org.apache.flink.runtime.rpc.MainThreadValidatorUtil;
@@ -76,9 +77,23 @@ class AkkaRpcActor<C extends RpcGateway, T extends RpcEndpoint<C>> extends Untyp
/** the helper that tracks whether calls come from the main thread */
private final MainThreadValidatorUtil mainThreadValidator;
- AkkaRpcActor(final T rpcEndpoint) {
+ private final CompletableFuture<Void> terminationFuture;
+
+ AkkaRpcActor(final T rpcEndpoint, final CompletableFuture<Void> terminationFuture) {
this.rpcEndpoint = checkNotNull(rpcEndpoint, "rpc endpoint");
this.mainThreadValidator = new MainThreadValidatorUtil(rpcEndpoint);
+ this.terminationFuture = checkNotNull(terminationFuture);
+ }
+
+ @Override
+ public void postStop() {
+ super.postStop();
+
+ // IMPORTANT: This only works if we don't use a restarting supervisor strategy. Otherwise
+ // we would complete the future and let the actor system restart the actor with a completed
+ // future.
+ // Complete the termination future so that others know that we've stopped.
+ terminationFuture.complete(null);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/1a9e6447/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
index fb7896a..44719c8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcService.java
@@ -32,9 +32,12 @@ import akka.dispatch.Mapper;
import akka.pattern.Patterns;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.concurrent.CompletableFuture;
import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
import org.apache.flink.runtime.concurrent.impl.FlinkFuture;
import org.apache.flink.runtime.rpc.MainThreadExecutable;
+import org.apache.flink.runtime.rpc.SelfGateway;
import org.apache.flink.runtime.rpc.RpcGateway;
import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcService;
@@ -131,7 +134,12 @@ public class AkkaRpcService implements RpcService {
final String address = AkkaUtils.getAkkaURL(actorSystem, actorRef);
- InvocationHandler akkaInvocationHandler = new AkkaInvocationHandler(address, actorRef, timeout, maximumFramesize);
+ InvocationHandler akkaInvocationHandler = new AkkaInvocationHandler(
+ address,
+ actorRef,
+ timeout,
+ maximumFramesize,
+ null);
// Rather than using the System ClassLoader directly, we derive the ClassLoader
// from this class . That works better in cases where Flink runs embedded and all Flink
@@ -156,7 +164,8 @@ public class AkkaRpcService implements RpcService {
public <C extends RpcGateway, S extends RpcEndpoint<C>> C startServer(S rpcEndpoint) {
checkNotNull(rpcEndpoint, "rpc endpoint");
- Props akkaRpcActorProps = Props.create(AkkaRpcActor.class, rpcEndpoint);
+ CompletableFuture<Void> terminationFuture = new FlinkCompletableFuture<>();
+ Props akkaRpcActorProps = Props.create(AkkaRpcActor.class, rpcEndpoint, terminationFuture);
ActorRef actorRef;
synchronized (lock) {
@@ -169,7 +178,12 @@ public class AkkaRpcService implements RpcService {
final String address = AkkaUtils.getAkkaURL(actorSystem, actorRef);
- InvocationHandler akkaInvocationHandler = new AkkaInvocationHandler(address, actorRef, timeout, maximumFramesize);
+ InvocationHandler akkaInvocationHandler = new AkkaInvocationHandler(
+ address,
+ actorRef,
+ timeout,
+ maximumFramesize,
+ terminationFuture);
// Rather than using the System ClassLoader directly, we derive the ClassLoader
// from this class . That works better in cases where Flink runs embedded and all Flink
@@ -181,6 +195,7 @@ public class AkkaRpcService implements RpcService {
classLoader,
new Class<?>[]{
rpcEndpoint.getSelfGatewayType(),
+ SelfGateway.class,
MainThreadExecutable.class,
StartStoppable.class,
AkkaGateway.class},
@@ -231,6 +246,17 @@ public class AkkaRpcService implements RpcService {
}
@Override
+ public Future<Void> getTerminationFuture() {
+ return FlinkFuture.supplyAsync(new Callable<Void>(){
+ @Override
+ public Void call() throws Exception {
+ actorSystem.awaitTermination();
+ return null;
+ }
+ }, getExecutor());
+ }
+
+ @Override
public Executor getExecutor() {
return actorSystem.dispatcher();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1a9e6447/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java
index 2a004c5..88906a7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/TestingSerialRpcService.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.rpc;
import akka.dispatch.Futures;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.runtime.concurrent.CompletableFuture;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
import org.apache.flink.runtime.util.DirectExecutorService;
@@ -39,7 +40,6 @@ import java.util.concurrent.TimeUnit;
import static org.apache.flink.util.Preconditions.checkNotNull;
-
/**
* An RPC Service implementation for testing. This RPC service directly executes all asynchronous
* calls one by one in the calling thread.
@@ -48,10 +48,12 @@ public class TestingSerialRpcService implements RpcService {
private final DirectExecutorService executorService;
private final ConcurrentHashMap<String, RpcGateway> registeredConnections;
+ private final CompletableFuture<Void> terminationFuture;
public TestingSerialRpcService() {
executorService = new DirectExecutorService();
this.registeredConnections = new ConcurrentHashMap<>(16);
+ this.terminationFuture = new FlinkCompletableFuture<>();
}
@Override
@@ -89,6 +91,12 @@ public class TestingSerialRpcService implements RpcService {
public void stopService() {
executorService.shutdown();
registeredConnections.clear();
+ terminationFuture.complete(null);
+ }
+
+ @Override
+ public Future<Void> getTerminationFuture() {
+ return terminationFuture;
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/1a9e6447/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
index 5d76024..ba8eb11 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
@@ -22,6 +22,8 @@ import akka.actor.ActorSystem;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.concurrent.impl.FlinkFuture;
import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcGateway;
import org.apache.flink.runtime.rpc.RpcMethod;
@@ -32,9 +34,15 @@ import org.hamcrest.core.Is;
import org.junit.AfterClass;
import org.junit.Test;
+import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -148,6 +156,34 @@ public class AkkaRpcActorTest extends TestLogger {
}
}
+ /**
+ * Tests that we can wait for a RpcEndpoint to terminate.
+ *
+ * @throws ExecutionException
+ * @throws InterruptedException
+ */
+ @Test(timeout=1000)
+ public void testRpcEndpointTerminationFuture() throws ExecutionException, InterruptedException {
+ final DummyRpcEndpoint rpcEndpoint = new DummyRpcEndpoint(akkaRpcService);
+ rpcEndpoint.start();
+
+ Future<Void> terminationFuture = rpcEndpoint.getTerminationFuture();
+
+ assertFalse(terminationFuture.isDone());
+
+ FlinkFuture.supplyAsync(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ rpcEndpoint.shutDown();
+
+ return null;
+ }
+ }, actorSystem.dispatcher());
+
+ // wait until the rpc endpoint has terminated
+ terminationFuture.get();
+ }
+
private interface DummyRpcGateway extends RpcGateway {
Future<Integer> foobar();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1a9e6447/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceTest.java
index 3388011..7c8defa 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcServiceTest.java
@@ -23,6 +23,7 @@ import org.apache.flink.api.common.time.Time;
import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.impl.FlinkFuture;
import org.apache.flink.util.TestLogger;
import org.junit.AfterClass;
@@ -34,6 +35,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class AkkaRpcServiceTest extends TestLogger {
@@ -120,4 +122,31 @@ public class AkkaRpcServiceTest extends TestLogger {
public void testGetAddress() {
assertEquals(AkkaUtils.getAddress(actorSystem).host().get(), akkaRpcService.getAddress());
}
+
+ /**
+ * Tests that we can wait for the termination of the rpc service
+ *
+ * @throws ExecutionException
+ * @throws InterruptedException
+ */
+ @Test(timeout = 1000)
+ public void testTerminationFuture() throws ExecutionException, InterruptedException {
+ final ActorSystem actorSystem = AkkaUtils.createDefaultActorSystem();
+ final AkkaRpcService rpcService = new AkkaRpcService(actorSystem, Time.milliseconds(1000));
+
+ Future<Void> terminationFuture = rpcService.getTerminationFuture();
+
+ assertFalse(terminationFuture.isDone());
+
+ FlinkFuture.supplyAsync(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ rpcService.stopService();
+
+ return null;
+ }
+ }, actorSystem.dispatcher());
+
+ terminationFuture.get();
+ }
}
[43/50] [abbrv] flink git commit: [FLINK-4847] Let
RpcEndpoint.start/shutDown throw exceptions
Posted by tr...@apache.org.
[FLINK-4847] Let RpcEndpoint.start/shutDown throw exceptions
Allowing the RpcEndpoint.start/shutDown to throw exceptions will help to let rpc endpoints
to quickly fail without having to use a callback like the FatalErrorHandler.
This closes #2651.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/0262e27e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/0262e27e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/0262e27e
Branch: refs/heads/flip-6
Commit: 0262e27e6ebd551c20b93b3bc55d7a8f4f72f70a
Parents: d2d0b12
Author: Till Rohrmann <tr...@apache.org>
Authored: Mon Oct 17 16:22:16 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:34 2016 +0100
----------------------------------------------------------------------
.../flink/runtime/jobmaster/JobManagerRunner.java | 6 +++++-
.../org/apache/flink/runtime/jobmaster/JobMaster.java | 14 ++++++++++----
.../org/apache/flink/runtime/rpc/RpcEndpoint.java | 8 ++++++--
.../flink/runtime/taskexecutor/TaskExecutor.java | 2 +-
.../flink/runtime/taskexecutor/TaskManagerRunner.java | 2 +-
.../flink/runtime/rpc/akka/AkkaRpcActorTest.java | 7 +------
.../runtime/rpc/akka/MainThreadValidationTest.java | 2 +-
.../runtime/rpc/akka/MessageSerializationTest.java | 2 +-
.../flink/runtime/taskexecutor/TaskExecutorTest.java | 4 ++--
9 files changed, 28 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/0262e27e/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
index 3313d8a..9d8e004 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
@@ -374,7 +374,11 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions, F
// This will eventually be noticed, but can not be ruled out from the beginning.
if (leaderElectionService.hasLeadership()) {
if (jobRunning) {
- jobManager.start(leaderSessionID);
+ try {
+ jobManager.start(leaderSessionID);
+ } catch (Exception e) {
+ onFatalError(new Exception("Could not start the job manager.", e));
+ }
} else {
log.info("Job {} ({}) already finished by others.", jobGraph.getName(), jobGraph.getJobID());
jobFinishedByOther();
http://git-wip-us.apache.org/repos/asf/flink/blob/0262e27e/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index a9ac1fe..49c4df1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -266,7 +266,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
*
* @param leaderSessionID The necessary leader id for running the job.
*/
- public void start(final UUID leaderSessionID) {
+ public void start(final UUID leaderSessionID) throws Exception {
if (LEADER_ID_UPDATER.compareAndSet(this, null, leaderSessionID)) {
super.start();
@@ -282,7 +282,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
* Suspend the job and shutdown all other services including rpc.
*/
@Override
- public void shutDown() {
+ public void shutDown() throws Exception {
// make sure there is a graceful exit
getSelf().suspendExecution(new Exception("JobManager is shutting down."));
super.shutDown();
@@ -381,7 +381,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
try {
resourceManagerLeaderRetriever.stop();
} catch (Exception e) {
- log.warn("Failed to stop resource manager leader retriever when suspending.");
+ log.warn("Failed to stop resource manager leader retriever when suspending.", e);
}
closeResourceManagerConnection();
@@ -767,7 +767,13 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
@Override
public void run() {
log.error("Fatal error occurred on JobManager, cause: {}", cause.getMessage(), cause);
- shutDown();
+
+ try {
+ shutDown();
+ } catch (Exception e) {
+ cause.addSuppressed(e);
+ }
+
errorHandler.onFatalError(cause);
}
});
http://git-wip-us.apache.org/repos/asf/flink/blob/0262e27e/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
index f93a2e2..b971b96 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/RpcEndpoint.java
@@ -110,8 +110,10 @@ public abstract class RpcEndpoint<C extends RpcGateway> {
*
* IMPORTANT: Whenever you override this method, call the parent implementation to enable
* rpc processing. It is advised to make the parent call last.
+ *
+ * @throws Exception indicating that something went wrong while starting the RPC endpoint
*/
- public void start() {
+ public void start() throws Exception {
((StartStoppable) self).start();
}
@@ -123,8 +125,10 @@ public abstract class RpcEndpoint<C extends RpcGateway> {
*
* <p>This method can be overridden to add RPC endpoint specific shut down code.
* The overridden method should always call the parent shut down method.
+ *
+ * @throws Exception indicating that the something went wrong while shutting the RPC endpoint down
*/
- public void shutDown() {
+ public void shutDown() throws Exception {
rpcService.stopServer(self);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0262e27e/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index 601d804..aea926c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -194,7 +194,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
// ------------------------------------------------------------------------
@Override
- public void start() {
+ public void start() throws Exception {
super.start();
// start by connecting to the ResourceManager
http://git-wip-us.apache.org/repos/asf/flink/blob/0262e27e/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
index f56d17c..7d9ee55 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
@@ -107,7 +107,7 @@ public class TaskManagerRunner implements FatalErrorHandler {
// Lifecycle management
// --------------------------------------------------------------------------------------------
- public void start() {
+ public void start() throws Exception {
taskManager.start();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0262e27e/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
index ba8eb11..d2dbab7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
@@ -22,7 +22,6 @@ import akka.actor.ActorSystem;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.concurrent.Future;
-import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
import org.apache.flink.runtime.concurrent.impl.FlinkFuture;
import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcGateway;
@@ -36,10 +35,6 @@ import org.junit.Test;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ForkJoinPool;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -163,7 +158,7 @@ public class AkkaRpcActorTest extends TestLogger {
* @throws InterruptedException
*/
@Test(timeout=1000)
- public void testRpcEndpointTerminationFuture() throws ExecutionException, InterruptedException {
+ public void testRpcEndpointTerminationFuture() throws Exception {
final DummyRpcEndpoint rpcEndpoint = new DummyRpcEndpoint(akkaRpcService);
rpcEndpoint.start();
http://git-wip-us.apache.org/repos/asf/flink/blob/0262e27e/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/MainThreadValidationTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/MainThreadValidationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/MainThreadValidationTest.java
index 9ec1f7e..9f134d8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/MainThreadValidationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/MainThreadValidationTest.java
@@ -34,7 +34,7 @@ import static org.junit.Assert.assertTrue;
public class MainThreadValidationTest extends TestLogger {
@Test
- public void failIfNotInMainThread() {
+ public void failIfNotInMainThread() throws Exception {
// test if assertions are activated. The test only works if assertions are loaded.
try {
assert false;
http://git-wip-us.apache.org/repos/asf/flink/blob/0262e27e/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/MessageSerializationTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/MessageSerializationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/MessageSerializationTest.java
index 0d5dc28..d640a97 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/MessageSerializationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/MessageSerializationTest.java
@@ -80,7 +80,7 @@ public class MessageSerializationTest extends TestLogger {
* Tests that a local rpc call with a non serializable argument can be executed.
*/
@Test
- public void testNonSerializableLocalMessageTransfer() throws InterruptedException, IOException {
+ public void testNonSerializableLocalMessageTransfer() throws Exception {
LinkedBlockingQueue<Object> linkedBlockingQueue = new LinkedBlockingQueue<>();
TestEndpoint testEndpoint = new TestEndpoint(akkaRpcService1, linkedBlockingQueue);
testEndpoint.start();
http://git-wip-us.apache.org/repos/asf/flink/blob/0262e27e/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index 87bde35..caae54e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -339,7 +339,7 @@ public class TaskExecutorTest extends TestLogger {
* the job leader, it will offer all reserved slots to the JobManager.
*/
@Test
- public void testJobLeaderDetection() throws TestingFatalErrorHandler.TestingException, SlotAllocationException {
+ public void testJobLeaderDetection() throws Exception {
final JobID jobId = new JobID();
final TestingSerialRpcService rpc = new TestingSerialRpcService();
@@ -606,7 +606,7 @@ public class TaskExecutorTest extends TestLogger {
*/
@Ignore
@Test
- public void testRejectAllocationRequestsForOutOfSyncSlots() throws SlotAllocationException {
+ public void testRejectAllocationRequestsForOutOfSyncSlots() throws Exception {
final ResourceID resourceID = ResourceID.generate();
final String address1 = "/resource/manager/address/one";
[15/50] [abbrv] flink git commit: [hotfix] Fix failing
JobManagerRunnerMockTest
Posted by tr...@apache.org.
[hotfix] Fix failing JobManagerRunnerMockTest
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/aac9137e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/aac9137e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/aac9137e
Branch: refs/heads/flip-6
Commit: aac9137e92b373645ff82f7f8e95f1f5c1f14acb
Parents: 36c3896
Author: Till Rohrmann <tr...@apache.org>
Authored: Tue Oct 4 14:27:37 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:29 2016 +0100
----------------------------------------------------------------------
.../jobmaster/JobManagerRunnerMockTest.java | 17 +----------------
1 file changed, 1 insertion(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/aac9137e/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
index 3a769bb..f8a0b6a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
@@ -131,21 +131,6 @@ public class JobManagerRunnerMockTest {
}
@Test
- public void testJobFinishedByOtherBeforeGrantLeadership() throws Exception {
- runner.start();
-
- when(submittedJobGraphStore.contains(any(JobID.class))).thenReturn(false);
- runner.grantLeadership(UUID.randomUUID());
-
- // runner should shutdown automatic and informed the job completion
- verify(leaderElectionService).stop();
- verify(jobManager).shutDown();
-
- assertTrue(jobCompletion.isJobFinished());
- assertTrue(jobCompletion.isJobFinishedByOther());
- }
-
- @Test
public void testJobFinished() throws Exception {
runner.start();
@@ -211,7 +196,7 @@ public class JobManagerRunnerMockTest {
UUID leaderSessionID2 = UUID.randomUUID();
runner.grantLeadership(leaderSessionID2);
- verify(jobManagerGateway, times(2)).startJob(leaderSessionID2);
+ verify(jobManagerGateway).startJob(leaderSessionID2);
}
private static class TestingOnCompletionActions implements OnCompletionActions {
[35/50] [abbrv] flink git commit: [FLINK-4835] [cluster management]
Add embedded version of the high-availability services
Posted by tr...@apache.org.
[FLINK-4835] [cluster management] Add embedded version of the high-availability services
This includes the addition of the EmbeddedLeaderService
and a clean shutdown hook for all high availability services.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5e719ca8
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5e719ca8
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5e719ca8
Branch: refs/heads/flip-6
Commit: 5e719ca8b262ca21dd20bfc8cf293cf307f9406d
Parents: b3e1a3d
Author: Stephan Ewen <se...@apache.org>
Authored: Fri Oct 14 23:57:11 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:32 2016 +0100
----------------------------------------------------------------------
.../StandaloneCheckpointRecoveryFactory.java | 4 +-
.../highavailability/EmbeddedNonHaServices.java | 62 +++
.../HighAvailabilityServices.java | 7 +-
.../runtime/highavailability/NonHaServices.java | 62 +--
.../highavailability/ZookeeperHaServices.java | 12 +-
.../nonha/AbstractNonHaServices.java | 175 +++++++
.../nonha/EmbeddedLeaderService.java | 466 +++++++++++++++++++
.../TestingHighAvailabilityServices.java | 9 +
8 files changed, 736 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/5e719ca8/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StandaloneCheckpointRecoveryFactory.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StandaloneCheckpointRecoveryFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StandaloneCheckpointRecoveryFactory.java
index a9624fb..57785ce 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StandaloneCheckpointRecoveryFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StandaloneCheckpointRecoveryFactory.java
@@ -40,8 +40,8 @@ public class StandaloneCheckpointRecoveryFactory implements CheckpointRecoveryFa
public CompletedCheckpointStore createCheckpointStore(JobID jobId, ClassLoader userClassLoader)
throws Exception {
- return new StandaloneCompletedCheckpointStore(CheckpointRecoveryFactory
- .NUMBER_OF_SUCCESSFUL_CHECKPOINTS_TO_RETAIN);
+ return new StandaloneCompletedCheckpointStore(
+ CheckpointRecoveryFactory.NUMBER_OF_SUCCESSFUL_CHECKPOINTS_TO_RETAIN);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/5e719ca8/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java
new file mode 100644
index 0000000..58da287
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.highavailability;
+
+import org.apache.flink.runtime.highavailability.nonha.AbstractNonHaServices;
+import org.apache.flink.runtime.highavailability.nonha.EmbeddedLeaderService;
+import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+
+/**
+ * An implementation of the {@link HighAvailabilityServices} for the non-high-availability case
+ * where all participants (ResourceManager, JobManagers, TaskManagers) run in the same process.
+ *
+ * <p>This implementation has no dependencies on any external services. It returns a fix
+ * pre-configured ResourceManager, and stores checkpoints and metadata simply on the heap or
+ * on a local file system and therefore in a storage without guarantees.
+ */
+public class EmbeddedNonHaServices extends AbstractNonHaServices implements HighAvailabilityServices {
+
+ private final EmbeddedLeaderService resourceManagerLeaderService;
+
+ public EmbeddedNonHaServices() {
+ super();
+ this.resourceManagerLeaderService = new EmbeddedLeaderService(getExecutorService());
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public LeaderRetrievalService getResourceManagerLeaderRetriever() throws Exception {
+ return resourceManagerLeaderService.createLeaderRetrievalService();
+ }
+
+ @Override
+ public LeaderElectionService getResourceManagerLeaderElectionService() throws Exception {
+ return resourceManagerLeaderService.createLeaderElectionService();
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public void shutdown() throws Exception {
+ super.shutdown();
+ resourceManagerLeaderService.shutdown();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5e719ca8/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
index 484cddb..f6db682 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
@@ -49,11 +49,10 @@ public interface HighAvailabilityServices {
* Gets the leader retriever for the job JobMaster which is responsible for the given job
*
* @param jobID The identifier of the job.
- * @param defaultAddress address under which the job manager is reachable
* @return
* @throws Exception
*/
- LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultAddress) throws Exception;
+ LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception;
/**
* Gets the leader election service for the cluster's resource manager.
@@ -86,4 +85,8 @@ public interface HighAvailabilityServices {
* Creates the BLOB store in which BLOBs are stored in a highly-available fashion.
*/
BlobStore createBlobStore() throws IOException;
+
+ // ------------------------------------------------------------------------
+
+ void shutdown() throws Exception;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5e719ca8/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
index 1c73c01..107cbd0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
@@ -18,21 +18,13 @@
package org.apache.flink.runtime.highavailability;
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.blob.BlobStore;
-import org.apache.flink.runtime.blob.VoidBlobStore;
-import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
-import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory;
-import org.apache.flink.runtime.highavailability.nonha.NonHaRegistry;
-import org.apache.flink.runtime.jobmanager.StandaloneSubmittedJobGraphStore;
-import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
+import org.apache.flink.runtime.highavailability.nonha.AbstractNonHaServices;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.leaderelection.StandaloneLeaderElectionService;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService;
import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -41,35 +33,23 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
* This implementation can be used for testing, and for cluster setups that do not
* tolerate failures of the master processes (JobManager, ResourceManager).
*
- * <p>This implementation has no dependencies on any external services. It returns fix
- * pre-configured leaders, and stores checkpoints and metadata simply on the heap and therefore
- * in volatile memory.
+ * <p>This implementation has no dependencies on any external services. It returns a fix
+ * pre-configured ResourceManager, and stores checkpoints and metadata simply on the heap or
+ * on a local file system and therefore in a storage without guarantees.
*/
-public class NonHaServices implements HighAvailabilityServices {
+public class NonHaServices extends AbstractNonHaServices implements HighAvailabilityServices {
/** The fix address of the ResourceManager */
private final String resourceManagerAddress;
- private final ConcurrentHashMap<JobID, String> jobMastersAddress;
-
/**
* Creates a new services class for the fix pre-defined leaders.
*
* @param resourceManagerAddress The fix address of the ResourceManager
*/
public NonHaServices(String resourceManagerAddress) {
+ super();
this.resourceManagerAddress = checkNotNull(resourceManagerAddress);
- this.jobMastersAddress = new ConcurrentHashMap<>(16);
- }
-
- /**
- * Binds address of a specified job master
- *
- * @param jobID JobID for the specified job master
- * @param jobMasterAddress address for the specified job master
- */
- public void bindJobMasterLeaderAddress(JobID jobID, String jobMasterAddress) {
- jobMastersAddress.put(jobID, jobMasterAddress);
}
// ------------------------------------------------------------------------
@@ -82,37 +62,7 @@ public class NonHaServices implements HighAvailabilityServices {
}
@Override
- public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultAddress) throws Exception {
- return new StandaloneLeaderRetrievalService(defaultAddress, new UUID(0, 0));
- }
-
- @Override
public LeaderElectionService getResourceManagerLeaderElectionService() throws Exception {
return new StandaloneLeaderElectionService();
}
-
- @Override
- public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) throws Exception {
- return new StandaloneLeaderElectionService();
- }
-
- @Override
- public CheckpointRecoveryFactory getCheckpointRecoveryFactory() throws Exception {
- return new StandaloneCheckpointRecoveryFactory();
- }
-
- @Override
- public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
- return new StandaloneSubmittedJobGraphStore();
- }
-
- @Override
- public RunningJobsRegistry getRunningJobsRegistry() throws Exception {
- return new NonHaRegistry();
- }
-
- @Override
- public BlobStore createBlobStore() {
- return new VoidBlobStore();
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5e719ca8/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
index bbe8ecb..eae45ab 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
@@ -85,7 +85,8 @@ public class ZookeeperHaServices implements HighAvailabilityServices {
private static final String JOB_MANAGER_LEADER_PATH = "/job_manager_lock";
// ------------------------------------------------------------------------
-
+
+
/** The ZooKeeper client to use */
private final CuratorFramework client;
@@ -168,6 +169,15 @@ public class ZookeeperHaServices implements HighAvailabilityServices {
}
// ------------------------------------------------------------------------
+ // Shutdown
+ // ------------------------------------------------------------------------
+
+ @Override
+ public void shutdown() throws Exception {
+ client.close();
+ }
+
+ // ------------------------------------------------------------------------
// Utilities
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/5e719ca8/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/AbstractNonHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/AbstractNonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/AbstractNonHaServices.java
new file mode 100644
index 0000000..8c15a52
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/AbstractNonHaServices.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.highavailability.nonha;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.blob.BlobStore;
+import org.apache.flink.runtime.blob.VoidBlobStore;
+import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
+import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.RunningJobsRegistry;
+import org.apache.flink.runtime.jobmanager.StandaloneSubmittedJobGraphStore;
+import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
+import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+
+import javax.annotation.Nonnull;
+import javax.annotation.concurrent.GuardedBy;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
+/**
+ * Base for all {@link HighAvailabilityServices} that are not highly available, but are backed
+ * by storage that has no availability guarantees and leader election services that cannot
+ * elect among multiple distributed leader contenders.
+ */
+public abstract class AbstractNonHaServices implements HighAvailabilityServices {
+
+ private final Object lock = new Object();
+
+ private final ExecutorService executor;
+
+ private final HashMap<JobID, EmbeddedLeaderService> jobManagerLeaderServices;
+
+ private final RunningJobsRegistry runningJobsRegistry;
+
+ private boolean shutdown;
+
+ // ------------------------------------------------------------------------
+
+ public AbstractNonHaServices() {
+ this.executor = Executors.newCachedThreadPool(new ServicesThreadFactory());
+ this.jobManagerLeaderServices = new HashMap<>();
+ this.runningJobsRegistry = new NonHaRegistry();
+ }
+
+ // ------------------------------------------------------------------------
+ // services
+ // ------------------------------------------------------------------------
+
+ @Override
+ public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception {
+ checkNotNull(jobID);
+
+ synchronized (lock) {
+ checkNotShutdown();
+ EmbeddedLeaderService service = getOrCreateJobManagerService(jobID);
+ return service.createLeaderRetrievalService();
+ }
+ }
+
+ @Override
+ public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) throws Exception {
+ checkNotNull(jobID);
+
+ synchronized (lock) {
+ checkNotShutdown();
+ EmbeddedLeaderService service = getOrCreateJobManagerService(jobID);
+ return service.createLeaderElectionService();
+ }
+ }
+
+ @GuardedBy("lock")
+ private EmbeddedLeaderService getOrCreateJobManagerService(JobID jobID) {
+ EmbeddedLeaderService service = jobManagerLeaderServices.get(jobID);
+ if (service == null) {
+ service = new EmbeddedLeaderService(executor);
+ jobManagerLeaderServices.put(jobID, service);
+ }
+ return service;
+ }
+
+ @Override
+ public CheckpointRecoveryFactory getCheckpointRecoveryFactory() throws Exception {
+ checkNotShutdown();
+ return new StandaloneCheckpointRecoveryFactory();
+ }
+
+ @Override
+ public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
+ checkNotShutdown();
+ return new StandaloneSubmittedJobGraphStore();
+ }
+
+ @Override
+ public RunningJobsRegistry getRunningJobsRegistry() throws Exception {
+ checkNotShutdown();
+ return runningJobsRegistry;
+ }
+
+ @Override
+ public BlobStore createBlobStore() throws IOException {
+ checkNotShutdown();
+ return new VoidBlobStore();
+ }
+
+ // ------------------------------------------------------------------------
+ // shutdown
+ // ------------------------------------------------------------------------
+
+ @Override
+ public void shutdown() throws Exception {
+ synchronized (lock) {
+ if (!shutdown) {
+ shutdown = true;
+
+ // no further calls should be dispatched
+ executor.shutdownNow();
+
+ // stop all job manager leader services
+ for (EmbeddedLeaderService service : jobManagerLeaderServices.values()) {
+ service.shutdown();
+ }
+ jobManagerLeaderServices.clear();
+ }
+ }
+ }
+
+ private void checkNotShutdown() {
+ checkState(!shutdown, "high availability services are shut down");
+ }
+
+ // ------------------------------------------------------------------------
+ // utilities
+ // ------------------------------------------------------------------------
+
+ protected ExecutorService getExecutorService() {
+ return executor;
+ }
+
+ private static final class ServicesThreadFactory implements ThreadFactory {
+
+ private AtomicInteger enumerator = new AtomicInteger();
+
+ @Override
+ public Thread newThread(@Nonnull Runnable r) {
+ Thread thread = new Thread(r, "Flink HA services thread #" + enumerator.incrementAndGet());
+ thread.setDaemon(true);
+ return thread;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5e719ca8/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/EmbeddedLeaderService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/EmbeddedLeaderService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/EmbeddedLeaderService.java
new file mode 100644
index 0000000..84ac551
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/EmbeddedLeaderService.java
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.highavailability.nonha;
+
+import org.apache.flink.runtime.leaderelection.LeaderContender;
+import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.GuardedBy;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
+/**
+ * A simple leader election service, which selects a leader among contenders and notifies listeners.
+ *
+ * <p>An election service for contenders can be created via {@link #createLeaderElectionService()},
+ * a listener service for leader observers can be created via {@link #createLeaderRetrievalService()}.
+ */
+public class EmbeddedLeaderService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(EmbeddedLeaderService.class);
+
+ private final Object lock = new Object();
+
+ private final Executor notificationExecutor;
+
+ private final Set<EmbeddedLeaderElectionService> allLeaderContenders;
+
+ private final Set<EmbeddedLeaderRetrievalService> listeners;
+
+ /** proposed leader, which has been notified of leadership grant, but has not confirmed */
+ private EmbeddedLeaderElectionService currentLeaderProposed;
+
+ /** actual leader that has confirmed leadership and of which listeners have been notified */
+ private EmbeddedLeaderElectionService currentLeaderConfirmed;
+
+ /** fencing UID for the current leader (or proposed leader) */
+ private UUID currentLeaderSessionId;
+
+ /** the cached address of the current leader */
+ private String currentLeaderAddress;
+
+ /** flag marking the service as terminated */
+ private boolean shutdown;
+
+ // ------------------------------------------------------------------------
+
+ public EmbeddedLeaderService(ExecutorService notificationsDispatcher) {
+ this.notificationExecutor = checkNotNull(notificationsDispatcher);
+ this.allLeaderContenders = new HashSet<>();
+ this.listeners = new HashSet<>();
+ }
+
+ // ------------------------------------------------------------------------
+ // shutdown and errors
+ // ------------------------------------------------------------------------
+
+ /**
+ * Shuts down this leader election service.
+ *
+ * <p>This method does not perform a clean revocation of the leader status and
+ * no notification to any leader listeners. It simply notifies all contenders
+ * and listeners that the service is no longer available.
+ */
+ public void shutdown() {
+ synchronized (lock) {
+ shutdownInternally(new Exception("Leader election service is shutting down"));
+ }
+ }
+
+ private void fatalError(Throwable error) {
+ LOG.error("Embedded leader election service encountered a fatal error. Shutting down service.", error);
+
+ synchronized (lock) {
+ shutdownInternally(new Exception("Leader election service is shutting down after a fatal error", error));
+ }
+ }
+
+ @GuardedBy("lock")
+ private void shutdownInternally(Exception exceptionForHandlers) {
+ assert Thread.holdsLock(lock);
+
+ if (!shutdown) {
+ // clear all leader status
+ currentLeaderProposed = null;
+ currentLeaderConfirmed = null;
+ currentLeaderSessionId = null;
+ currentLeaderAddress = null;
+
+ // fail all registered listeners
+ for (EmbeddedLeaderElectionService service : allLeaderContenders) {
+ service.shutdown(exceptionForHandlers);
+ }
+ allLeaderContenders.clear();
+
+ // fail all registered listeners
+ for (EmbeddedLeaderRetrievalService service : listeners) {
+ service.shutdown(exceptionForHandlers);
+ }
+ listeners.clear();
+
+ shutdown = true;
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // creating contenders and listeners
+ // ------------------------------------------------------------------------
+
+ public LeaderElectionService createLeaderElectionService() {
+ checkState(!shutdown, "leader election service is shut down");
+ return new EmbeddedLeaderElectionService();
+ }
+
+ public LeaderRetrievalService createLeaderRetrievalService() {
+ checkState(!shutdown, "leader election service is shut down");
+ return new EmbeddedLeaderRetrievalService();
+ }
+
+ // ------------------------------------------------------------------------
+ // adding and removing contenders & listeners
+ // ------------------------------------------------------------------------
+
+ /**
+ * Callback from leader contenders when they start their service.
+ */
+ void addContender(EmbeddedLeaderElectionService service, LeaderContender contender) {
+ synchronized (lock) {
+ checkState(!shutdown, "leader election service is shut down");
+ checkState(!service.running, "leader election service is already started");
+
+ try {
+ if (!allLeaderContenders.add(service)) {
+ throw new IllegalStateException("leader election service was added to this service multiple times");
+ }
+
+ service.contender = contender;
+ service.running = true;
+
+ updateLeader();
+ }
+ catch (Throwable t) {
+ fatalError(t);
+ }
+ }
+ }
+
+ /**
+ * Callback from leader contenders when they stop their service.
+ */
+ void removeContender(EmbeddedLeaderElectionService service) {
+ synchronized (lock) {
+ // if the service was not even started, simply do nothing
+ if (!service.running || shutdown) {
+ return;
+ }
+
+ try {
+ if (!allLeaderContenders.remove(service)) {
+ throw new IllegalStateException("leader election service does not belong to this service");
+ }
+
+ // stop the service
+ service.contender = null;
+ service.running = false;
+ service.isLeader = false;
+
+ // if that was the current leader, unset its status
+ if (currentLeaderConfirmed == service) {
+ currentLeaderConfirmed = null;
+ currentLeaderSessionId = null;
+ currentLeaderAddress = null;
+ }
+ if (currentLeaderProposed == service) {
+ currentLeaderProposed = null;
+ currentLeaderSessionId = null;
+ }
+
+ updateLeader();
+ }
+ catch (Throwable t) {
+ fatalError(t);
+ }
+ }
+ }
+
+ /**
+ * Callback from leader contenders when they confirm a leader grant
+ */
+ void confirmLeader(final EmbeddedLeaderElectionService service, final UUID leaderSessionId) {
+ synchronized (lock) {
+ // if the service was shut down in the meantime, ignore this confirmation
+ if (!service.running || shutdown) {
+ return;
+ }
+
+ try {
+ // check if the confirmation is for the same grant, or whether it is a stale grant
+ if (service == currentLeaderProposed && currentLeaderSessionId.equals(leaderSessionId)) {
+ final String address = service.contender.getAddress();
+ LOG.info("Received confirmation of leadership for leader {} / session={}", address, leaderSessionId);
+
+ // mark leadership
+ currentLeaderConfirmed = service;
+ currentLeaderAddress = address;
+ currentLeaderProposed = null;
+ service.isLeader = true;
+
+ // notify all listeners
+ for (EmbeddedLeaderRetrievalService listener : listeners) {
+ notificationExecutor.execute(
+ new NotifyOfLeaderCall(address, leaderSessionId, listener.listener, LOG));
+ }
+ }
+ else {
+ LOG.debug("Received confirmation of leadership for a stale leadership grant. Ignoring.");
+ }
+ }
+ catch (Throwable t) {
+ fatalError(t);
+ }
+ }
+ }
+
+ @GuardedBy("lock")
+ private void updateLeader() {
+ // this must be called under the lock
+ assert Thread.holdsLock(lock);
+
+ if (currentLeaderConfirmed == null && currentLeaderProposed == null) {
+ // we need a new leader
+ if (allLeaderContenders.isEmpty()) {
+ // no new leader available, tell everyone that there is no leader currently
+ for (EmbeddedLeaderRetrievalService listener : listeners) {
+ notificationExecutor.execute(
+ new NotifyOfLeaderCall(null, null, listener.listener, LOG));
+ }
+ }
+ else {
+ // propose a leader and ask it
+ final UUID leaderSessionId = UUID.randomUUID();
+ EmbeddedLeaderElectionService leaderService = allLeaderContenders.iterator().next();
+
+ currentLeaderSessionId = leaderSessionId;
+ currentLeaderProposed = leaderService;
+
+ notificationExecutor.execute(
+ new GrantLeadershipCall(leaderService.contender, leaderSessionId, LOG));
+ }
+ }
+ }
+
+ void addListener(EmbeddedLeaderRetrievalService service, LeaderRetrievalListener listener) {
+ synchronized (lock) {
+ checkState(!shutdown, "leader election service is shut down");
+ checkState(!service.running, "leader retrieval service is already started");
+
+ try {
+ if (!listeners.add(service)) {
+ throw new IllegalStateException("leader retrieval service was added to this service multiple times");
+ }
+
+ service.listener = listener;
+ service.running = true;
+
+ // if we already have a leader, immediately notify this new listener
+ if (currentLeaderConfirmed != null) {
+ notificationExecutor.execute(
+ new NotifyOfLeaderCall(currentLeaderAddress, currentLeaderSessionId, listener, LOG));
+ }
+ }
+ catch (Throwable t) {
+ fatalError(t);
+ }
+ }
+ }
+
+ void removeListener(EmbeddedLeaderRetrievalService service) {
+ synchronized (lock) {
+ // if the service was not even started, simply do nothing
+ if (!service.running || shutdown) {
+ return;
+ }
+
+ try {
+ if (!listeners.remove(service)) {
+ throw new IllegalStateException("leader retrieval service does not belong to this service");
+ }
+
+ // stop the service
+ service.listener = null;
+ service.running = false;
+ }
+ catch (Throwable t) {
+ fatalError(t);
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // election and retrieval service implementations
+ // ------------------------------------------------------------------------
+
+ private class EmbeddedLeaderElectionService implements LeaderElectionService {
+
+ volatile LeaderContender contender;
+
+ volatile boolean isLeader;
+
+ volatile boolean running;
+
+ @Override
+ public void start(LeaderContender contender) throws Exception {
+ checkNotNull(contender);
+ addContender(this, contender);
+ }
+
+ @Override
+ public void stop() throws Exception {
+ removeContender(this);
+ }
+
+ @Override
+ public void confirmLeaderSessionID(UUID leaderSessionID) {
+ checkNotNull(leaderSessionID);
+ confirmLeader(this, leaderSessionID);
+ }
+
+ @Override
+ public boolean hasLeadership() {
+ return isLeader;
+ }
+
+ void shutdown(Exception cause) {
+ if (running) {
+ running = false;
+ isLeader = false;
+ contender.handleError(cause);
+ contender = null;
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ private class EmbeddedLeaderRetrievalService implements LeaderRetrievalService {
+
+ volatile LeaderRetrievalListener listener;
+
+ volatile boolean running;
+
+ @Override
+ public void start(LeaderRetrievalListener listener) throws Exception {
+ checkNotNull(listener);
+ addListener(this, listener);
+ }
+
+ @Override
+ public void stop() throws Exception {
+ removeListener(this);
+ }
+
+ public void shutdown(Exception cause) {
+ if (running) {
+ running = false;
+ listener.handleError(cause);
+ listener = null;
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // asynchronous notifications
+ // ------------------------------------------------------------------------
+
+ private static class NotifyOfLeaderCall implements Runnable {
+
+ @Nullable
+ private final String address; // null if leader revoked without new leader
+ @Nullable
+ private final UUID leaderSessionId; // null if leader revoked without new leader
+
+ private final LeaderRetrievalListener listener;
+ private final Logger logger;
+
+ NotifyOfLeaderCall(
+ @Nullable String address,
+ @Nullable UUID leaderSessionId,
+ LeaderRetrievalListener listener,
+ Logger logger) {
+
+ this.address = address;
+ this.leaderSessionId = leaderSessionId;
+ this.listener = checkNotNull(listener);
+ this.logger = checkNotNull(logger);
+ }
+
+ @Override
+ public void run() {
+ try {
+ listener.notifyLeaderAddress(address, leaderSessionId);
+ }
+ catch (Throwable t) {
+ logger.warn("Error notifying leader listener about new leader", t);
+ listener.handleError(t instanceof Exception ? (Exception) t : new Exception(t));
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ private static class GrantLeadershipCall implements Runnable {
+
+ private final LeaderContender contender;
+ private final UUID leaderSessionId;
+ private final Logger logger;
+
+ GrantLeadershipCall(
+ LeaderContender contender,
+ UUID leaderSessionId,
+ Logger logger) {
+
+ this.contender = checkNotNull(contender);
+ this.leaderSessionId = checkNotNull(leaderSessionId);
+ this.logger = checkNotNull(logger);
+ }
+
+ @Override
+ public void run() {
+ try {
+ contender.grantLeadership(leaderSessionId);
+ }
+ catch (Throwable t) {
+ logger.warn("Error notifying leader listener about new leader", t);
+ contender.handleError(t instanceof Exception ? (Exception) t : new Exception(t));
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5e719ca8/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
index 38e372d..3e88e8c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
@@ -154,4 +154,13 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
public BlobStore createBlobStore() throws IOException {
return new VoidBlobStore();
}
+
+ // ------------------------------------------------------------------------
+ // Shutdown
+ // ------------------------------------------------------------------------
+
+ @Override
+ public void shutdown() throws Exception {
+ // nothing to do, since this should not shut down individual services, but cross service parts
+ }
}
[17/50] [abbrv] flink git commit: [FLINK-4347][FLINK-4348] simplify
SlotManager and integrate it with ResourceManager
Posted by tr...@apache.org.
[FLINK-4347][FLINK-4348] simplify SlotManager and integrate it with ResourceManager
Instead of relying on a full synchronization of all slots information on
every heartbeat, the SlotManager is now responsible for updating its
state. It initially syncs all slots upon registration of the
TaskExecutor. After that, it only receives notifications from the
TaskExecutor when slots become available again. This simplifies the
logic of the SlotManager and makes the slot allocation more predictable
in case of message loss.
Additional changes:
- Move the slot registration and allocation report to the registration
of the TaskExecutor
- Let the TaskExecutor immediately notify the ResourceManager once a
slot becomes free. The ResourceManager has to confirm this
notification. Otherwise, the slot will be blocked because the
ResourceManager's state is not in sync.
- Integrate with handleSlotRequestFailedAtTaskManager and introduce
fencing to protect against TaskExecutors which are not registered
anymore.
- introduce RPC call to notify ResourceManager about free slots
- ignore out-of-date slot requests from ResourceManager at TaskExecutor
- let the ResourceManager update its state instead of relying on heartbeats
- provide ResourceManagerServices to SlotManager
- introduce factory for SlotManager
- keep task gateways and worker information in ResourceManager and
inform SlotManager
- add TaskExecutor test to ensure that a free slot which hasn't been
confirmed by the task executor is correctly blacklisted as long as the
ResourceManager has not confirmed the allocation removal.
- adapt tests
- update javadocs
This closes #2571.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8adceede
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8adceede
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8adceede
Branch: refs/heads/flip-6
Commit: 8adceedee8390168055c699547301a3e21397e33
Parents: aac9137
Author: Maximilian Michels <mx...@apache.org>
Authored: Thu Sep 29 15:08:32 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:29 2016 +0100
----------------------------------------------------------------------
.../clusterframework/types/ResourceSlot.java | 12 +-
.../resourcemanager/ResourceManager.java | 115 +++++--
.../resourcemanager/ResourceManagerGateway.java | 31 +-
.../ResourceManagerServices.java | 2 +-
.../resourcemanager/SlotRequestRegistered.java | 33 --
.../resourcemanager/SlotRequestRejected.java | 34 --
.../resourcemanager/SlotRequestReply.java | 41 ---
.../StandaloneResourceManager.java | 8 +-
.../jobmanager/RMSlotRequestRegistered.java | 33 ++
.../jobmanager/RMSlotRequestRejected.java | 34 ++
.../messages/jobmanager/RMSlotRequestReply.java | 41 +++
.../taskexecutor/SlotAvailableReply.java | 47 +++
.../taskexecutor/TMSlotRequestRegistered.java | 35 ++
.../taskexecutor/TMSlotRequestRejected.java | 35 ++
.../taskexecutor/TMSlotRequestReply.java | 58 ++++
.../registration/TaskExecutorRegistration.java | 12 +-
.../registration/WorkerRegistration.java | 42 +++
.../slotmanager/SimpleSlotManager.java | 53 ---
.../slotmanager/SlotManager.java | 326 +++++++------------
.../slotmanager/SlotManagerFactory.java | 31 ++
.../flink/runtime/taskexecutor/SlotReport.java | 19 +-
.../runtime/taskexecutor/TaskExecutor.java | 37 ++-
.../taskexecutor/TaskExecutorGateway.java | 7 +-
...TaskExecutorToResourceManagerConnection.java | 2 +-
.../resourcemanager/ResourceManagerHATest.java | 12 +-
.../ResourceManagerJobMasterTest.java | 4 +-
.../ResourceManagerTaskExecutorTest.java | 53 +--
.../resourcemanager/TestingResourceManager.java | 53 +++
.../resourcemanager/TestingSlotManager.java | 78 +++++
.../slotmanager/SlotManagerTest.java | 239 ++++++--------
.../slotmanager/SlotProtocolTest.java | 92 ++++--
.../runtime/taskexecutor/TaskExecutorTest.java | 96 +++++-
32 files changed, 1087 insertions(+), 628 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceSlot.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceSlot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceSlot.java
index 4a91a79..0b9367d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceSlot.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/ResourceSlot.java
@@ -18,7 +18,7 @@
package org.apache.flink.runtime.clusterframework.types;
-import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
+import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorRegistration;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -35,12 +35,12 @@ public class ResourceSlot implements ResourceIDRetrievable {
private final ResourceProfile resourceProfile;
/** Gateway to the TaskExecutor which owns the slot */
- private final TaskExecutorGateway taskExecutorGateway;
+ private final TaskExecutorRegistration taskExecutorRegistration;
- public ResourceSlot(SlotID slotId, ResourceProfile resourceProfile, TaskExecutorGateway taskExecutorGateway) {
+ public ResourceSlot(SlotID slotId, ResourceProfile resourceProfile, TaskExecutorRegistration taskExecutorRegistration) {
this.slotId = checkNotNull(slotId);
this.resourceProfile = checkNotNull(resourceProfile);
- this.taskExecutorGateway = taskExecutorGateway;
+ this.taskExecutorRegistration = checkNotNull(taskExecutorRegistration);
}
@Override
@@ -56,8 +56,8 @@ public class ResourceSlot implements ResourceIDRetrievable {
return resourceProfile;
}
- public TaskExecutorGateway getTaskExecutorGateway() {
- return taskExecutorGateway;
+ public TaskExecutorRegistration getTaskExecutorRegistration() {
+ return taskExecutorRegistration;
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index f45afa3..d2d00cf 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -24,8 +24,8 @@ import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.clusterframework.ApplicationStatus;
import org.apache.flink.runtime.clusterframework.messages.InfoMessage;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.clusterframework.types.ResourceIDRetrievable;
import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import org.apache.flink.runtime.clusterframework.types.SlotID;
import org.apache.flink.runtime.concurrent.AcceptFunction;
import org.apache.flink.runtime.concurrent.ApplyFunction;
import org.apache.flink.runtime.concurrent.BiFunction;
@@ -37,7 +37,12 @@ import org.apache.flink.runtime.leaderelection.LeaderContender;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestRejected;
+import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestReply;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.SlotAvailableReply;
+import org.apache.flink.runtime.resourcemanager.registration.WorkerRegistration;
import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
@@ -45,12 +50,14 @@ import org.apache.flink.runtime.jobmaster.JobMaster;
import org.apache.flink.runtime.jobmaster.JobMasterGateway;
import org.apache.flink.runtime.registration.RegistrationResponse;
+import org.apache.flink.runtime.taskexecutor.SlotReport;
import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
import org.apache.flink.runtime.taskexecutor.TaskExecutorRegistrationSuccess;
import org.apache.flink.runtime.util.LeaderConnectionInfo;
import org.apache.flink.runtime.util.LeaderRetrievalUtils;
import scala.concurrent.duration.FiniteDuration;
+import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
@@ -70,41 +77,54 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
* <li>{@link #requestSlot(UUID, UUID, SlotRequest)} requests a slot from the resource manager</li>
* </ul>
*/
-public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
+public abstract class ResourceManager<WorkerType extends Serializable>
extends RpcEndpoint<ResourceManagerGateway>
implements LeaderContender {
- /** The exit code with which the process is stopped in case of a fatal error */
+ /** The exit code with which the process is stopped in case of a fatal error. */
protected static final int EXIT_CODE_FATAL_ERROR = -13;
+ /** All currently registered JobMasterGateways scoped by JobID. */
private final Map<JobID, JobMasterGateway> jobMasterGateways;
+ /** LeaderListeners for all registered JobMasters. */
private final Map<JobID, JobMasterLeaderListener> jobMasterLeaderRetrievalListeners;
- private final Map<ResourceID, WorkerType> taskExecutorGateways;
+ /** All currently registered TaskExecutors with there framework specific worker information. */
+ private final Map<ResourceID, WorkerRegistration<WorkerType>> taskExecutors;
+ /** High availability services for leader retrieval and election. */
private final HighAvailabilityServices highAvailabilityServices;
- private final SlotManager slotManager;
+ /** The factory to construct the SlotManager. */
+ private final SlotManagerFactory slotManagerFactory;
+ /** The SlotManager created by the slotManagerFactory when the ResourceManager is started. */
+ private SlotManager slotManager;
+
+ /** The service to elect a ResourceManager leader. */
private LeaderElectionService leaderElectionService;
+ /** ResourceManager's leader session id which is updated on leader election. */
private UUID leaderSessionID;
+ /** All registered listeners for status updates of the ResourceManager. */
private Map<String, InfoMessageListenerRpcGateway> infoMessageListeners;
+ /** Default timeout for messages */
private final Time timeout = Time.seconds(5);
public ResourceManager(
RpcService rpcService,
HighAvailabilityServices highAvailabilityServices,
- SlotManager slotManager) {
+ SlotManagerFactory slotManagerFactory) {
super(rpcService);
this.highAvailabilityServices = checkNotNull(highAvailabilityServices);
+ this.slotManagerFactory = checkNotNull(slotManagerFactory);
this.jobMasterGateways = new HashMap<>();
- this.slotManager = checkNotNull(slotManager);
this.jobMasterLeaderRetrievalListeners = new HashMap<>();
- this.taskExecutorGateways = new HashMap<>();
+ this.taskExecutors = new HashMap<>();
+ this.leaderSessionID = new UUID(0, 0);
infoMessageListeners = new HashMap<>();
}
@@ -113,9 +133,10 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
// start a leader
try {
super.start();
+ // SlotManager should start first
+ slotManager = slotManagerFactory.create(createResourceManagerServices());
leaderElectionService = highAvailabilityServices.getResourceManagerLeaderElectionService();
leaderElectionService.start(this);
- slotManager.setupResourceManagerServices(new DefaultResourceManagerServices());
// framework specific initialization
initialize();
} catch (Throwable e) {
@@ -196,7 +217,7 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
.handleAsync(new BiFunction<JobMasterGateway, Throwable, RegistrationResponse>() {
@Override
public RegistrationResponse apply(JobMasterGateway jobMasterGateway, Throwable throwable) {
-
+
if (throwable != null) {
return new RegistrationResponse.Decline(throwable.getMessage());
} else {
@@ -234,7 +255,8 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
public Future<RegistrationResponse> registerTaskExecutor(
final UUID resourceManagerLeaderId,
final String taskExecutorAddress,
- final ResourceID resourceID) {
+ final ResourceID resourceID,
+ final SlotReport slotReport) {
return getRpcService().execute(new Callable<TaskExecutorGateway>() {
@Override
@@ -245,7 +267,8 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
resourceID, taskExecutorAddress, leaderSessionID, resourceManagerLeaderId);
throw new Exception("Invalid leader session id");
}
- return getRpcService().connect(taskExecutorAddress, TaskExecutorGateway.class).get(5, TimeUnit.SECONDS);
+ return getRpcService().connect(taskExecutorAddress, TaskExecutorGateway.class)
+ .get(timeout.toMilliseconds(), timeout.getUnit());
}
}).handleAsync(new BiFunction<TaskExecutorGateway, Throwable, RegistrationResponse>() {
@Override
@@ -253,14 +276,17 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
if (throwable != null) {
return new RegistrationResponse.Decline(throwable.getMessage());
} else {
- WorkerType oldWorker = taskExecutorGateways.remove(resourceID);
- if (oldWorker != null) {
+ WorkerRegistration oldRegistration = taskExecutors.remove(resourceID);
+ if (oldRegistration != null) {
// TODO :: suggest old taskExecutor to stop itself
- slotManager.notifyTaskManagerFailure(resourceID);
+ log.info("Replacing old instance of worker for ResourceID {}", resourceID);
}
WorkerType newWorker = workerStarted(resourceID);
- taskExecutorGateways.put(resourceID, newWorker);
- return new TaskExecutorRegistrationSuccess(new InstanceID(), 5000);
+ WorkerRegistration<WorkerType> registration =
+ new WorkerRegistration<>(taskExecutorGateway, newWorker);
+ taskExecutors.put(resourceID, registration);
+ slotManager.registerTaskExecutor(resourceID, registration, slotReport);
+ return new TaskExecutorRegistrationSuccess(registration.getInstanceID(), 5000);
}
}
}, getMainThreadExecutor());
@@ -273,7 +299,7 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
* @return Slot assignment
*/
@RpcMethod
- public SlotRequestReply requestSlot(
+ public RMSlotRequestReply requestSlot(
UUID jobMasterLeaderID,
UUID resourceManagerLeaderID,
SlotRequest slotRequest) {
@@ -290,8 +316,41 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
return slotManager.requestSlot(slotRequest);
} else {
log.info("Ignoring slot request for unknown JobMaster with JobID {}", jobId);
- return new SlotRequestRejected(slotRequest.getAllocationId());
+ return new RMSlotRequestRejected(slotRequest.getAllocationId());
+ }
+ }
+
+ /**
+ * Notification from a TaskExecutor that a slot has become available
+ * @param resourceManagerLeaderId TaskExecutor's resource manager leader id
+ * @param resourceID TaskExecutor's resource id
+ * @param instanceID TaskExecutor's instance id
+ * @param slotID The slot id of the available slot
+ * @return SlotAvailableReply
+ */
+ @RpcMethod
+ public SlotAvailableReply notifySlotAvailable(
+ final UUID resourceManagerLeaderId,
+ final ResourceID resourceID,
+ final InstanceID instanceID,
+ final SlotID slotID) {
+
+ if (resourceManagerLeaderId.equals(leaderSessionID)) {
+ WorkerRegistration<WorkerType> registration = taskExecutors.get(resourceID);
+ if (registration != null) {
+ InstanceID registrationInstanceID = registration.getInstanceID();
+ if (registrationInstanceID.equals(instanceID)) {
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ slotManager.notifySlotAvailable(resourceID, slotID);
+ }
+ });
+ return new SlotAvailableReply(leaderSessionID, slotID);
+ }
+ }
}
+ return null;
}
@@ -329,9 +388,9 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
public void run() {
log.info("ResourceManager {} was revoked leadership.", getAddress());
jobMasterGateways.clear();
- taskExecutorGateways.clear();
+ taskExecutors.clear();
slotManager.clearState();
- leaderSessionID = null;
+ leaderSessionID = new UUID(0, 0);
}
});
}
@@ -411,7 +470,7 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
runAsync(new Runnable() {
@Override
public void run() {
- WorkerType worker = taskExecutorGateways.remove(resourceID);
+ WorkerType worker = taskExecutors.remove(resourceID).getWorker();
if (worker != null) {
// TODO :: suggest failed task executor to stop itself
slotManager.notifyTaskManagerFailure(resourceID);
@@ -426,7 +485,7 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
* @return The number of currently started TaskManagers.
*/
public int getNumberOfStartedTaskManagers() {
- return taskExecutorGateways.size();
+ return taskExecutors.size();
}
/**
@@ -507,6 +566,14 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
});
}
+ // ------------------------------------------------------------------------
+ // Resource Manager Services
+ // ------------------------------------------------------------------------
+
+ protected ResourceManagerServices createResourceManagerServices() {
+ return new DefaultResourceManagerServices();
+ }
+
private class DefaultResourceManagerServices implements ResourceManagerServices {
@Override
@@ -520,7 +587,7 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
}
@Override
- public Executor getExecutor() {
+ public Executor getMainThreadExecutor() {
return ResourceManager.this.getMainThreadExecutor();
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
index 87303a1..3c81227 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
@@ -22,11 +22,16 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.clusterframework.ApplicationStatus;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.clusterframework.types.SlotID;
import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.instance.InstanceID;
+import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestReply;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.SlotAvailableReply;
import org.apache.flink.runtime.rpc.RpcGateway;
import org.apache.flink.runtime.rpc.RpcTimeout;
import org.apache.flink.runtime.jobmaster.JobMaster;
import org.apache.flink.runtime.registration.RegistrationResponse;
+import org.apache.flink.runtime.taskexecutor.SlotReport;
import java.util.UUID;
@@ -56,10 +61,12 @@ public interface ResourceManagerGateway extends RpcGateway {
/**
* Requests a slot from the resource manager.
*
- * @param slotRequest Slot request
- * @return Future slot assignment
+ * @param jobMasterLeaderID leader id of the JobMaster
+ * @param resourceManagerLeaderID leader if of the ResourceMaster
+ * @param slotRequest The slot to request
+ * @return The confirmation that the slot gets allocated
*/
- Future<SlotRequestReply> requestSlot(
+ Future<RMSlotRequestReply> requestSlot(
UUID jobMasterLeaderID,
UUID resourceManagerLeaderID,
SlotRequest slotRequest,
@@ -71,6 +78,7 @@ public interface ResourceManagerGateway extends RpcGateway {
* @param resourceManagerLeaderId The fencing token for the ResourceManager leader
* @param taskExecutorAddress The address of the TaskExecutor that registers
* @param resourceID The resource ID of the TaskExecutor that registers
+ * @param slotReport The slot report containing free and allocated task slots
* @param timeout The timeout for the response.
*
* @return The future to the response by the ResourceManager.
@@ -79,6 +87,23 @@ public interface ResourceManagerGateway extends RpcGateway {
UUID resourceManagerLeaderId,
String taskExecutorAddress,
ResourceID resourceID,
+ SlotReport slotReport,
+ @RpcTimeout Time timeout);
+
+ /**
+ * Sent by the TaskExecutor to notify the ResourceManager that a slot has become available.
+ *
+ * @param resourceManagerLeaderId The ResourceManager leader id
+ * @param resourceID The ResourceID of the TaskExecutor
+ * @param instanceID The InstanceID of the TaskExecutor
+ * @param slotID The SlotID of the freed slot
+ * @return The confirmation by the ResourceManager
+ */
+ Future<SlotAvailableReply> notifySlotAvailable(
+ UUID resourceManagerLeaderId,
+ ResourceID resourceID,
+ InstanceID instanceID,
+ SlotID slotID,
@RpcTimeout Time timeout);
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
index 30994dc..b997a3a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
@@ -39,6 +39,6 @@ public interface ResourceManagerServices {
/**
* Gets the executor which executes in the main thread of the ResourceManager
*/
- Executor getExecutor();
+ Executor getMainThreadExecutor();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequestRegistered.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequestRegistered.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequestRegistered.java
deleted file mode 100644
index f719dce..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequestRegistered.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.resourcemanager;
-
-import org.apache.flink.runtime.clusterframework.types.AllocationID;
-
-/**
- * Acknowledgment by the ResourceManager for a SlotRequest from the JobManager
- */
-public class SlotRequestRegistered extends SlotRequestReply {
-
- private static final long serialVersionUID = 4760320859275256855L;
-
- public SlotRequestRegistered(AllocationID allocationID) {
- super(allocationID);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequestRejected.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequestRejected.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequestRejected.java
deleted file mode 100644
index 282a7d5..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequestRejected.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.resourcemanager;
-
-import org.apache.flink.runtime.clusterframework.types.AllocationID;
-
-/**
- * Rejection message by the ResourceManager for a SlotRequest from the JobManager
- */
-public class SlotRequestRejected extends SlotRequestReply {
-
- private static final long serialVersionUID = 9049346740895325144L;
-
- public SlotRequestRejected(AllocationID allocationID) {
- super(allocationID);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequestReply.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequestReply.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequestReply.java
deleted file mode 100644
index 1b85d0c..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/SlotRequestReply.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.resourcemanager;
-
-import org.apache.flink.runtime.clusterframework.types.AllocationID;
-
-import java.io.Serializable;
-
-/**
- * Acknowledgment by the ResourceManager for a SlotRequest from the JobManager
- */
-public abstract class SlotRequestReply implements Serializable {
-
- private static final long serialVersionUID = 42;
-
- private final AllocationID allocationID;
-
- public SlotRequestReply(AllocationID allocationID) {
- this.allocationID = allocationID;
- }
-
- public AllocationID getAllocationID() {
- return allocationID;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
index deca8d3..f9f55f8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
@@ -22,7 +22,7 @@ import org.apache.flink.runtime.clusterframework.ApplicationStatus;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
import org.apache.flink.runtime.rpc.RpcService;
/**
@@ -34,9 +34,9 @@ import org.apache.flink.runtime.rpc.RpcService;
public class StandaloneResourceManager extends ResourceManager<ResourceID> {
public StandaloneResourceManager(RpcService rpcService,
- HighAvailabilityServices highAvailabilityServices,
- SlotManager slotManager) {
- super(rpcService, highAvailabilityServices, slotManager);
+ HighAvailabilityServices highAvailabilityServices,
+ SlotManagerFactory slotManagerFactory) {
+ super(rpcService, highAvailabilityServices, slotManagerFactory);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRegistered.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRegistered.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRegistered.java
new file mode 100644
index 0000000..01bc532
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRegistered.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.messages.jobmanager;
+
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+
+/**
+ * Acknowledgment by the ResourceManager for a SlotRequest from the JobManager
+ */
+public class RMSlotRequestRegistered extends RMSlotRequestReply {
+
+ private static final long serialVersionUID = 4760320859275256855L;
+
+ public RMSlotRequestRegistered(AllocationID allocationID) {
+ super(allocationID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRejected.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRejected.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRejected.java
new file mode 100644
index 0000000..649d61c
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestRejected.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.messages.jobmanager;
+
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+
+/**
+ * Rejection message by the ResourceManager for a SlotRequest from the JobManager
+ */
+public class RMSlotRequestRejected extends RMSlotRequestReply {
+
+ private static final long serialVersionUID = 9049346740895325144L;
+
+ public RMSlotRequestRejected(AllocationID allocationID) {
+ super(allocationID);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestReply.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestReply.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestReply.java
new file mode 100644
index 0000000..66e1911
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/jobmanager/RMSlotRequestReply.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.messages.jobmanager;
+
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+
+import java.io.Serializable;
+
+/**
+ * Acknowledgment by the ResourceManager for a SlotRequest from the JobManager
+ */
+public abstract class RMSlotRequestReply implements Serializable {
+
+ private static final long serialVersionUID = 42;
+
+ private final AllocationID allocationID;
+
+ public RMSlotRequestReply(AllocationID allocationID) {
+ this.allocationID = allocationID;
+ }
+
+ public AllocationID getAllocationID() {
+ return allocationID;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/SlotAvailableReply.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/SlotAvailableReply.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/SlotAvailableReply.java
new file mode 100644
index 0000000..f2e0105
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/SlotAvailableReply.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.runtime.resourcemanager.messages.taskexecutor;
+
+import org.apache.flink.runtime.clusterframework.types.SlotID;
+
+import java.io.Serializable;
+import java.util.UUID;
+
+/**
+ * Sent by the ResourceManager to the TaskExecutor confirm receipt of
+ * {@code org.apache.flink.runtime.resourcemanager.ResourceManagerGateway.notifySlotAvailable}.
+ */
+public class SlotAvailableReply implements Serializable {
+
+ private final UUID resourceManagerLeaderID;
+
+ private final SlotID slotID;
+
+ public SlotAvailableReply(UUID resourceManagerLeaderID, SlotID slotID) {
+ this.resourceManagerLeaderID = resourceManagerLeaderID;
+ this.slotID = slotID;
+ }
+
+ public UUID getResourceManagerLeaderID() {
+ return resourceManagerLeaderID;
+ }
+
+ public SlotID getSlotID() {
+ return slotID;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRegistered.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRegistered.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRegistered.java
new file mode 100644
index 0000000..c0f0f49
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRegistered.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.messages.taskexecutor;
+
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.instance.InstanceID;
+
+/**
+ * Acknowledgment by the TaskExecutor for a SlotRequest from the ResourceManager
+ */
+public class TMSlotRequestRegistered extends TMSlotRequestReply {
+
+ private static final long serialVersionUID = 4760320859275256855L;
+
+ public TMSlotRequestRegistered(InstanceID instanceID, ResourceID resourceID, AllocationID allocationID) {
+ super(instanceID, resourceID, allocationID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRejected.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRejected.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRejected.java
new file mode 100644
index 0000000..9b10a35
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestRejected.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.messages.taskexecutor;
+
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.instance.InstanceID;
+
+/**
+ * Rejection by the TaskExecutor for a SlotRequest from the ResourceManager
+ */
+public class TMSlotRequestRejected extends TMSlotRequestReply {
+
+ private static final long serialVersionUID = 9049346740895325144L;
+
+ public TMSlotRequestRejected(InstanceID instanceID, ResourceID resourceID, AllocationID allocationID) {
+ super(instanceID, resourceID, allocationID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestReply.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestReply.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestReply.java
new file mode 100644
index 0000000..b23b6e9
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/messages/taskexecutor/TMSlotRequestReply.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.messages.taskexecutor;
+
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.instance.InstanceID;
+
+import java.io.Serializable;
+
+/**
+ * Acknowledgment by the TaskExecutor for a SlotRequest from the ResourceManager
+ */
+public abstract class TMSlotRequestReply implements Serializable {
+
+ private static final long serialVersionUID = 42;
+
+ private final InstanceID instanceID;
+
+ private final ResourceID resourceID;
+
+ private final AllocationID allocationID;
+
+ protected TMSlotRequestReply(InstanceID instanceID, ResourceID resourceID, AllocationID allocationID) {
+ this.instanceID = instanceID;
+ this.resourceID = resourceID;
+ this.allocationID = allocationID;
+ }
+
+ public InstanceID getInstanceID() {
+ return instanceID;
+ }
+
+ public ResourceID getResourceID() {
+ return resourceID;
+ }
+
+ public AllocationID getAllocationID() {
+ return allocationID;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorRegistration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorRegistration.java
index 6b21f5c..bfa9c00 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorRegistration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorRegistration.java
@@ -24,20 +24,20 @@ import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
import java.io.Serializable;
/**
- * This class is responsible for group the TaskExecutorGateway and the InstanceID of a registered task executor.
+ * This class is responsible for grouping the TaskExecutorGateway and the InstanceID
+ * of a registered task executor.
*/
public class TaskExecutorRegistration implements Serializable {
private static final long serialVersionUID = -2062957799469434614L;
- private TaskExecutorGateway taskExecutorGateway;
+ private final InstanceID instanceID;
- private InstanceID instanceID;
+ private TaskExecutorGateway taskExecutorGateway;
- public TaskExecutorRegistration(TaskExecutorGateway taskExecutorGateway,
- InstanceID instanceID) {
+ public TaskExecutorRegistration(TaskExecutorGateway taskExecutorGateway) {
+ this.instanceID = new InstanceID();
this.taskExecutorGateway = taskExecutorGateway;
- this.instanceID = instanceID;
}
public InstanceID getInstanceID() {
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/WorkerRegistration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/WorkerRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/WorkerRegistration.java
new file mode 100644
index 0000000..ff28f94
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/WorkerRegistration.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.registration;
+
+import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
+
+import java.io.Serializable;
+
+/**
+ * This class extends the {@link TaskExecutorRegistration}, adding the worker information.
+ */
+public class WorkerRegistration<WorkerType extends Serializable> extends TaskExecutorRegistration {
+
+ private static final long serialVersionUID = -2062957799469434614L;
+
+ private WorkerType worker;
+
+ public WorkerRegistration(TaskExecutorGateway taskExecutorGateway, WorkerType worker) {
+ super(taskExecutorGateway);
+ this.worker = worker;
+ }
+
+ public WorkerType getWorker() {
+ return worker;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SimpleSlotManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SimpleSlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SimpleSlotManager.java
deleted file mode 100644
index ae1de5a..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SimpleSlotManager.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.runtime.resourcemanager.slotmanager;
-
-import org.apache.flink.runtime.clusterframework.types.AllocationID;
-import org.apache.flink.runtime.clusterframework.types.ResourceSlot;
-import org.apache.flink.runtime.clusterframework.types.SlotID;
-import org.apache.flink.runtime.resourcemanager.SlotRequest;
-
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- * A simple SlotManager which ignores resource profiles.
- */
-public class SimpleSlotManager extends SlotManager {
-
- @Override
- protected ResourceSlot chooseSlotToUse(SlotRequest request, Map<SlotID, ResourceSlot> freeSlots) {
- final Iterator<ResourceSlot> slotIterator = freeSlots.values().iterator();
- if (slotIterator.hasNext()) {
- return slotIterator.next();
- } else {
- return null;
- }
- }
-
- @Override
- protected SlotRequest chooseRequestToFulfill(ResourceSlot offeredSlot, Map<AllocationID, SlotRequest> pendingRequests) {
- final Iterator<SlotRequest> requestIterator = pendingRequests.values().iterator();
- if (requestIterator.hasNext()) {
- return requestIterator.next();
- } else {
- return null;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
index a56b2f6..7eb2d78 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
@@ -28,11 +28,12 @@ import org.apache.flink.runtime.concurrent.BiFunction;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.resourcemanager.ResourceManagerServices;
import org.apache.flink.runtime.resourcemanager.SlotRequest;
-import org.apache.flink.runtime.resourcemanager.SlotRequestRegistered;
-import org.apache.flink.runtime.resourcemanager.SlotRequestReply;
+import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestRegistered;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRejected;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
+import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorRegistration;
import org.apache.flink.runtime.taskexecutor.SlotReport;
import org.apache.flink.runtime.taskexecutor.SlotStatus;
-import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,8 +65,11 @@ public abstract class SlotManager {
protected final Logger LOG = LoggerFactory.getLogger(getClass());
+ /** The Resource allocation provider */
+ protected final ResourceManagerServices rmServices;
+
/** All registered task managers with ResourceID and gateway. */
- private final Map<ResourceID, TaskExecutorGateway> taskManagerGateways;
+ private final Map<ResourceID, TaskExecutorRegistration> taskManagers;
/** All registered slots, including free and allocated slots */
private final Map<ResourceID, Map<SlotID, ResourceSlot>> registeredSlots;
@@ -84,29 +88,17 @@ public abstract class SlotManager {
/** The current leader id set by the ResourceManager */
private UUID leaderID;
- /** The Resource allocation provider */
- private ResourceManagerServices resourceManagerServices;
-
- public SlotManager() {
+ public SlotManager(ResourceManagerServices rmServices) {
+ this.rmServices = checkNotNull(rmServices);
this.registeredSlots = new HashMap<>(16);
this.pendingSlotRequests = new LinkedHashMap<>(16);
this.freeSlots = new HashMap<>(16);
this.allocationMap = new AllocationMap();
- this.taskManagerGateways = new HashMap<>();
+ this.taskManagers = new HashMap<>();
this.timeout = Time.seconds(10);
+ this.leaderID = new UUID(0, 0);
}
- /**
- * Initializes the resource supplier which is needed to request new resources.
- */
- public void setupResourceManagerServices(ResourceManagerServices resourceManagerServices) {
- if (this.resourceManagerServices != null) {
- throw new IllegalStateException("ResourceManagerServices may only be set once.");
- }
- this.resourceManagerServices = resourceManagerServices;
- }
-
-
// ------------------------------------------------------------------------
// slot managements
// ------------------------------------------------------------------------
@@ -118,13 +110,13 @@ public abstract class SlotManager {
* RPC's main thread to avoid race condition).
*
* @param request The detailed request of the slot
- * @return SlotRequestRegistered The confirmation message to be send to the caller
+ * @return RMSlotRequestRegistered The confirmation message to be send to the caller
*/
- public SlotRequestRegistered requestSlot(final SlotRequest request) {
+ public RMSlotRequestRegistered requestSlot(final SlotRequest request) {
final AllocationID allocationId = request.getAllocationId();
if (isRequestDuplicated(request)) {
LOG.warn("Duplicated slot request, AllocationID:{}", allocationId);
- return null;
+ return new RMSlotRequestRegistered(allocationId);
}
// try to fulfil the request with current free slots
@@ -136,53 +128,38 @@ public abstract class SlotManager {
// record this allocation in bookkeeping
allocationMap.addAllocation(slot.getSlotId(), allocationId);
// remove selected slot from free pool
- final ResourceSlot removedSlot = freeSlots.remove(slot.getSlotId());
-
- final Future<SlotRequestReply> slotRequestReplyFuture =
- slot.getTaskExecutorGateway().requestSlot(allocationId, leaderID, timeout);
-
- slotRequestReplyFuture.handleAsync(new BiFunction<SlotRequestReply, Throwable, Object>() {
- @Override
- public Object apply(SlotRequestReply slotRequestReply, Throwable throwable) {
- if (throwable != null) {
- // we failed, put the slot and the request back again
- if (allocationMap.isAllocated(slot.getSlotId())) {
- // only re-add if the slot hasn't been removed in the meantime
- freeSlots.put(slot.getSlotId(), removedSlot);
- }
- pendingSlotRequests.put(allocationId, request);
- }
- return null;
- }
- }, resourceManagerServices.getExecutor());
+ freeSlots.remove(slot.getSlotId());
+
+ sendSlotRequest(slot, request);
} else {
LOG.info("Cannot fulfil slot request, try to allocate a new container for it, " +
"AllocationID:{}, JobID:{}", allocationId, request.getJobId());
- Preconditions.checkState(resourceManagerServices != null,
+ Preconditions.checkState(rmServices != null,
"Attempted to allocate resources but no ResourceManagerServices set.");
- resourceManagerServices.allocateResource(request.getResourceProfile());
+ rmServices.allocateResource(request.getResourceProfile());
pendingSlotRequests.put(allocationId, request);
}
- return new SlotRequestRegistered(allocationId);
+ return new RMSlotRequestRegistered(allocationId);
}
/**
- * Sync slot status with TaskManager's SlotReport.
+ * Notifies the SlotManager that a slot is available again after being allocated.
+ * @param slotID slot id of available slot
*/
- public void updateSlotStatus(final SlotReport slotReport) {
- for (SlotStatus slotStatus : slotReport.getSlotsStatus()) {
- updateSlotStatus(slotStatus);
+ public void notifySlotAvailable(ResourceID resourceID, SlotID slotID) {
+ if (!allocationMap.isAllocated(slotID)) {
+ throw new IllegalStateException("Slot was not previously allocated but " +
+ "TaskManager reports it as available again");
}
- }
-
- /**
- * Registers a TaskExecutor
- * @param resourceID TaskExecutor's ResourceID
- * @param gateway TaskExcutor's gateway
- */
- public void registerTaskExecutor(ResourceID resourceID, TaskExecutorGateway gateway) {
- this.taskManagerGateways.put(resourceID, gateway);
+ allocationMap.removeAllocation(slotID);
+ final Map<SlotID, ResourceSlot> slots = registeredSlots.get(resourceID);
+ ResourceSlot freeSlot = slots.get(slotID);
+ if (freeSlot == null) {
+ throw new IllegalStateException("Slot was not registered with SlotManager but " +
+ "TaskManager reported it to be available.");
+ }
+ handleFreeSlot(freeSlot);
}
/**
@@ -200,51 +177,72 @@ public abstract class SlotManager {
* @param originalRequest The original slot request
* @param slotId The target SlotID
*/
- public void handleSlotRequestFailedAtTaskManager(final SlotRequest originalRequest, final SlotID slotId) {
+ void handleSlotRequestFailedAtTaskManager(final SlotRequest originalRequest, final SlotID slotId) {
final AllocationID originalAllocationId = originalRequest.getAllocationId();
LOG.info("Slot request failed at TaskManager, SlotID:{}, AllocationID:{}, JobID:{}",
slotId, originalAllocationId, originalRequest.getJobId());
- // verify the allocation info before we do anything
- if (freeSlots.containsKey(slotId)) {
- // this slot is currently empty, no need to de-allocate it from our allocations
- LOG.info("Original slot is somehow empty, retrying this request");
+ if (allocationMap.isAllocated(slotId)) {
+ final AllocationID expectedAllocationId = allocationMap.getAllocationID(slotId);
- // before retry, we should double check whether this request was allocated by some other ways
- if (!allocationMap.isAllocated(originalAllocationId)) {
- requestSlot(originalRequest);
+ // check whether we have an agreement on whom this slot belongs to
+ if (originalAllocationId.equals(expectedAllocationId)) {
+ LOG.info("De-allocate this request and retry");
+ allocationMap.removeAllocation(expectedAllocationId);
+ pendingSlotRequests.put(originalRequest.getAllocationId(), originalRequest);
+ ResourceSlot slot = checkNotNull(getRegisteredSlot(slotId));
+ // treat this slot as empty and retry with a different request
+ handleFreeSlot(slot);
} else {
- LOG.info("The failed request has somehow been allocated, SlotID:{}",
- allocationMap.getSlotID(originalAllocationId));
+ LOG.error("Slot request failed for slot {} with allocation id {}:" +
+ " Allocation id did not match the expected allocation id {}.",
+ slotId, originalAllocationId, expectedAllocationId);
}
- } else if (allocationMap.isAllocated(slotId)) {
- final AllocationID currentAllocationId = allocationMap.getAllocationID(slotId);
+ } else {
+ LOG.error("Slot request failed for slot {} with allocation id {}: " +
+ "Slot was not previously registered.",
+ slotId, originalAllocationId);
+ }
+ }
- // check whether we have an agreement on whom this slot belongs to
- if (originalAllocationId.equals(currentAllocationId)) {
- LOG.info("De-allocate this request and retry");
- allocationMap.removeAllocation(currentAllocationId);
+ /**
+ * Registers a TaskExecutor
+ * @param resourceID TaskExecutor's ResourceID
+ * @param registration TaskExecutor's registration
+ * @param slotReport TaskExecutor's free and allocated slots
+ */
+ public void registerTaskExecutor(
+ ResourceID resourceID,
+ TaskExecutorRegistration registration,
+ SlotReport slotReport) {
- // put this slot back to free pool
- ResourceSlot slot = checkNotNull(getRegisteredSlot(slotId));
- freeSlots.put(slotId, slot);
+ if (taskManagers.get(resourceID) != null) {
+ notifyTaskManagerFailure(resourceID);
+ }
- // retry the request
- requestSlot(originalRequest);
- } else {
- // the slot is taken by someone else, no need to de-allocate it from our allocations
- LOG.info("Original slot is taken by someone else, current AllocationID:{}", currentAllocationId);
+ this.taskManagers.put(resourceID, registration);
- // before retry, we should double check whether this request was allocated by some other ways
- if (!allocationMap.isAllocated(originalAllocationId)) {
- requestSlot(originalRequest);
- } else {
- LOG.info("The failed request is somehow been allocated, SlotID:{}",
- allocationMap.getSlotID(originalAllocationId));
- }
+ for (SlotStatus slotStatus : slotReport.getSlotsStatus()) {
+ final SlotID slotId = slotStatus.getSlotID();
+
+ final TaskExecutorRegistration taskExecutorRegistration = taskManagers.get(slotId.getResourceID());
+ if (taskExecutorRegistration == null) {
+ LOG.info("Received SlotStatus but ResourceID {} is unknown to the SlotManager",
+ slotId.getResourceID());
+ return;
+ }
+
+ final ResourceSlot slot = new ResourceSlot(slotId, slotStatus.getProfiler(), taskExecutorRegistration);
+
+ registerNewSlot(slot);
+ LOG.info("New slot appeared, SlotID:{}, AllocationID:{}", slotId, slotStatus.getAllocationID());
+
+ if (slotStatus.getAllocationID() != null) {
+ // slot in use, record this in bookkeeping
+ allocationMap.addAllocation(slotId, slotStatus.getAllocationID());
+ } else {
+ handleFreeSlot(slot);
}
- } else {
- LOG.error("BUG! {} is neither in free pool nor in allocated pool", slotId);
}
}
@@ -255,7 +253,7 @@ public abstract class SlotManager {
*/
public void notifyTaskManagerFailure(final ResourceID resourceId) {
LOG.info("Resource:{} been notified failure", resourceId);
- taskManagerGateways.remove(resourceId);
+ taskManagers.remove(resourceId);
final Map<SlotID, ResourceSlot> slotIdsToRemove = registeredSlots.remove(resourceId);
if (slotIdsToRemove != null) {
for (SlotID slotId : slotIdsToRemove.keySet()) {
@@ -276,92 +274,6 @@ public abstract class SlotManager {
// ------------------------------------------------------------------------
/**
- * Update slot status based on TaskManager's report. There are mainly two situations when we receive the report:
- * <ul>
- * <li>1. The slot is newly registered.</li>
- * <li>2. The slot has registered, it contains its current status.</li>
- * </ul>
- * <p>
- * Regarding 1: It's fairly simple, we just record this slot's status, and trigger schedule if slot is empty.
- * <p>
- * Regarding 2: It will cause some weird situation since we may have some time-gap on how the slot's status really
- * is. We may have some updates on the slot's allocation, but it doesn't reflected by TaskManager's heartbeat yet,
- * and we may make some wrong decision if we cannot guarantee we have the exact status about all the slots. So
- * the principle here is: We always trust TaskManager's heartbeat, we will correct our information based on that
- * and take next action based on the diff between our information and heartbeat status.
- *
- * @param reportedStatus Reported slot status
- */
- void updateSlotStatus(final SlotStatus reportedStatus) {
- final SlotID slotId = reportedStatus.getSlotID();
-
- final TaskExecutorGateway taskExecutorGateway = taskManagerGateways.get(slotId.getResourceID());
- if (taskExecutorGateway == null) {
- LOG.info("Received SlotStatus but ResourceID {} is unknown to the SlotManager",
- slotId.getResourceID());
- return;
- }
-
- final ResourceSlot slot = new ResourceSlot(slotId, reportedStatus.getProfiler(), taskExecutorGateway);
-
- if (registerNewSlot(slot)) {
- // we have a newly registered slot
- LOG.info("New slot appeared, SlotID:{}, AllocationID:{}", slotId, reportedStatus.getAllocationID());
-
- if (reportedStatus.getAllocationID() != null) {
- // slot in use, record this in bookkeeping
- allocationMap.addAllocation(slotId, reportedStatus.getAllocationID());
- } else {
- handleFreeSlot(slot);
- }
- } else {
- // slot exists, update current information
- if (reportedStatus.getAllocationID() != null) {
- // slot is reported in use
- final AllocationID reportedAllocationId = reportedStatus.getAllocationID();
-
- // check whether we also thought this slot is in use
- if (allocationMap.isAllocated(slotId)) {
- // we also think that slot is in use, check whether the AllocationID matches
- final AllocationID currentAllocationId = allocationMap.getAllocationID(slotId);
-
- if (!reportedAllocationId.equals(currentAllocationId)) {
- LOG.info("Slot allocation info mismatch! SlotID:{}, current:{}, reported:{}",
- slotId, currentAllocationId, reportedAllocationId);
-
- // seems we have a disagreement about the slot assignments, need to correct it
- allocationMap.removeAllocation(slotId);
- allocationMap.addAllocation(slotId, reportedAllocationId);
- }
- } else {
- LOG.info("Slot allocation info mismatch! SlotID:{}, current:null, reported:{}",
- slotId, reportedAllocationId);
-
- // we thought the slot is free, should correct this information
- allocationMap.addAllocation(slotId, reportedStatus.getAllocationID());
-
- // remove this slot from free slots pool
- freeSlots.remove(slotId);
- }
- } else {
- // slot is reported empty
-
- // check whether we also thought this slot is empty
- if (allocationMap.isAllocated(slotId)) {
- LOG.info("Slot allocation info mismatch! SlotID:{}, current:{}, reported:null",
- slotId, allocationMap.getAllocationID(slotId));
-
- // we thought the slot is in use, correct it
- allocationMap.removeAllocation(slotId);
-
- // we have a free slot!
- handleFreeSlot(slot);
- }
- }
- }
- }
-
- /**
* When we have a free slot, try to fulfill the pending request first. If any request can be fulfilled,
* record this allocation in bookkeeping and send slot request to TaskManager, else we just add this slot
* to the free pool.
@@ -373,32 +285,45 @@ public abstract class SlotManager {
if (chosenRequest != null) {
final AllocationID allocationId = chosenRequest.getAllocationId();
- final SlotRequest removedSlotRequest = pendingSlotRequests.remove(allocationId);
+ final SlotRequest slotRequest = pendingSlotRequests.remove(allocationId);
LOG.info("Assigning SlotID({}) to AllocationID({}), JobID:{}", freeSlot.getSlotId(),
allocationId, chosenRequest.getJobId());
allocationMap.addAllocation(freeSlot.getSlotId(), allocationId);
- final Future<SlotRequestReply> slotRequestReplyFuture =
- freeSlot.getTaskExecutorGateway().requestSlot(allocationId, leaderID, timeout);
-
- slotRequestReplyFuture.handleAsync(new BiFunction<SlotRequestReply, Throwable, Object>() {
- @Override
- public Object apply(SlotRequestReply slotRequestReply, Throwable throwable) {
- if (throwable != null) {
- // we failed, add the request back again
- if (allocationMap.isAllocated(freeSlot.getSlotId())) {
- pendingSlotRequests.put(allocationId, removedSlotRequest);
- }
- }
- return null;
- }
- }, resourceManagerServices.getExecutor());
+ sendSlotRequest(freeSlot, slotRequest);
} else {
freeSlots.put(freeSlot.getSlotId(), freeSlot);
}
}
+ private void sendSlotRequest(final ResourceSlot freeSlot, final SlotRequest slotRequest) {
+
+ final AllocationID allocationID = slotRequest.getAllocationId();
+ final TaskExecutorRegistration registration = freeSlot.getTaskExecutorRegistration();
+ final Future<TMSlotRequestReply> slotRequestReplyFuture =
+ registration.getTaskExecutorGateway()
+ .requestSlot(freeSlot.getSlotId(), allocationID, leaderID, timeout);
+
+ slotRequestReplyFuture.handleAsync(new BiFunction<TMSlotRequestReply, Throwable, Void>() {
+ @Override
+ public Void apply(TMSlotRequestReply slotRequestReply, Throwable throwable) {
+ TaskExecutorRegistration current = taskManagers.get(slotRequestReply.getResourceID());
+ if (current != null && current.getInstanceID().equals(slotRequestReply.getInstanceID())) {
+ if (throwable != null || slotRequestReply instanceof TMSlotRequestRejected) {
+ handleSlotRequestFailedAtTaskManager(slotRequest, freeSlot.getSlotId());
+ } else {
+ LOG.debug("Successfully registered slot {} ", freeSlot.getSlotId());
+ }
+ } else {
+ LOG.debug("Discarding message from obsolete TaskExecutor with InstanceID {}",
+ slotRequestReply.getInstanceID());
+ }
+ return null;
+ }
+ }, rmServices.getMainThreadExecutor());
+ }
+
/**
* Check whether the request is duplicated. We use AllocationID to identify slot request, for each
* formerly received slot request, it is either in pending list or already been allocated.
@@ -413,18 +338,17 @@ public abstract class SlotManager {
}
/**
- * Try to register slot, and tell if this slot is newly registered.
+ * Registers a new slot with the SlotManager.
*
- * @param slot The ResourceSlot which will be checked and registered
- * @return <tt>true</tt> if we meet a new slot
+ * @param slot The ResourceSlot which will be registered
*/
- private boolean registerNewSlot(final ResourceSlot slot) {
+ private void registerNewSlot(final ResourceSlot slot) {
final SlotID slotId = slot.getSlotId();
final ResourceID resourceId = slotId.getResourceID();
if (!registeredSlots.containsKey(resourceId)) {
registeredSlots.put(resourceId, new HashMap<SlotID, ResourceSlot>());
}
- return registeredSlots.get(resourceId).put(slotId, slot) == null;
+ registeredSlots.get(resourceId).put(slotId, slot);
}
private ResourceSlot getRegisteredSlot(final SlotID slotId) {
@@ -559,12 +483,12 @@ public abstract class SlotManager {
* Clears the state of the SlotManager after leadership revokal
*/
public void clearState() {
- taskManagerGateways.clear();
+ taskManagers.clear();
registeredSlots.clear();
pendingSlotRequests.clear();
freeSlots.clear();
allocationMap.clear();
- leaderID = null;
+ leaderID = new UUID(0, 0);
}
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerFactory.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerFactory.java
new file mode 100644
index 0000000..b4e9c99
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerFactory.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.runtime.resourcemanager.slotmanager;
+
+import org.apache.flink.runtime.resourcemanager.ResourceManagerServices;
+
+/**
+ * Factory to create a SlotManager and provide it with dependencies.
+ */
+public interface SlotManagerFactory {
+
+ /**
+ * Creates a SlotManager and provides it with ResourceManager services.
+ */
+ SlotManager create(ResourceManagerServices rmServices);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java
index a5de2d5..54adce6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/SlotReport.java
@@ -18,9 +18,8 @@
package org.apache.flink.runtime.taskexecutor;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
-
import java.io.Serializable;
+import java.util.Collections;
import java.util.List;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -37,20 +36,20 @@ public class SlotReport implements Serializable {
/** The slots status of the TaskManager */
private final List<SlotStatus> slotsStatus;
- /** The resource id which identifies the TaskManager */
- private final ResourceID resourceID;
+ public SlotReport() {
+ this(Collections.<SlotStatus>emptyList());
+ }
+
+ public SlotReport(SlotStatus slotStatus) {
+ this(Collections.singletonList(slotStatus));
+ }
- public SlotReport(final List<SlotStatus> slotsStatus, final ResourceID resourceID) {
+ public SlotReport(final List<SlotStatus> slotsStatus) {
this.slotsStatus = checkNotNull(slotsStatus);
- this.resourceID = checkNotNull(resourceID);
}
public List<SlotStatus> getSlotsStatus() {
return slotsStatus;
}
- public ResourceID getResourceID() {
- return resourceID;
- }
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index 7df0a91..c0041a3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -20,9 +20,12 @@ package org.apache.flink.runtime.taskexecutor;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.clusterframework.types.SlotID;
+import org.apache.flink.runtime.instance.InstanceID;
import org.apache.flink.runtime.metrics.MetricRegistry;
-import org.apache.flink.runtime.resourcemanager.SlotRequestRegistered;
-import org.apache.flink.runtime.resourcemanager.SlotRequestReply;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRegistered;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRejected;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.annotation.VisibleForTesting;
@@ -38,6 +41,8 @@ import org.apache.flink.runtime.rpc.RpcService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.HashSet;
+import java.util.Set;
import java.util.UUID;
import static org.apache.flink.util.Preconditions.checkArgument;
@@ -78,6 +83,9 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
/** The fatal error handler to use in case of a fatal error */
private final FatalErrorHandler fatalErrorHandler;
+ /** Slots which have become available but haven't been confirmed by the RM */
+ private final Set<SlotID> unconfirmedFreeSlots;
+
// --------- resource manager --------
private TaskExecutorToResourceManagerConnection resourceManagerConnection;
@@ -109,6 +117,8 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
this.numberOfSlots = taskManagerConfiguration.getNumberSlots();
+
+ this.unconfirmedFreeSlots = new HashSet<>();
}
// ------------------------------------------------------------------------
@@ -152,6 +162,8 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
+ unconfirmedFreeSlots.clear();
+
// establish a connection to the new leader
if (newLeaderAddress != null) {
log.info("Attempting to register at ResourceManager {}", newLeaderAddress);
@@ -169,13 +181,25 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
/**
* Requests a slot from the TaskManager
*
+ * @param slotID Slot id for the request
* @param allocationID id for the request
* @param resourceManagerLeaderID current leader id of the ResourceManager
* @return answer to the slot request
*/
@RpcMethod
- public SlotRequestReply requestSlot(AllocationID allocationID, UUID resourceManagerLeaderID) {
- return new SlotRequestRegistered(allocationID);
+ public TMSlotRequestReply requestSlot(SlotID slotID, AllocationID allocationID, UUID resourceManagerLeaderID) {
+ if (!resourceManagerConnection.getTargetLeaderId().equals(resourceManagerLeaderID)) {
+ return new TMSlotRequestRejected(
+ resourceManagerConnection.getRegistrationId(), getResourceID(), allocationID);
+ }
+ if (unconfirmedFreeSlots.contains(slotID)) {
+ // check if request has not been blacklisted because the notification of a free slot
+ // has not been confirmed by the ResourceManager
+ return new TMSlotRequestRejected(
+ resourceManagerConnection.getRegistrationId(), getResourceID(), allocationID);
+ }
+ return new TMSlotRequestRegistered(new InstanceID(), ResourceID.generate(), allocationID);
+
}
// ------------------------------------------------------------------------
@@ -227,6 +251,11 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
return resourceManagerConnection;
}
+ @VisibleForTesting
+ public void addUnconfirmedFreeSlotNotification(SlotID slotID) {
+ unconfirmedFreeSlots.add(slotID);
+ }
+
// ------------------------------------------------------------------------
// Utility classes
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
index 0962802..2360b53 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
@@ -20,8 +20,9 @@ package org.apache.flink.runtime.taskexecutor;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.SlotID;
import org.apache.flink.runtime.concurrent.Future;
-import org.apache.flink.runtime.resourcemanager.SlotRequestReply;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
import org.apache.flink.runtime.rpc.RpcGateway;
import org.apache.flink.runtime.rpc.RpcTimeout;
@@ -41,11 +42,13 @@ public interface TaskExecutorGateway extends RpcGateway {
/**
* Requests a slot from the TaskManager
*
+ * @param slotID slot id for the request
* @param allocationID id for the request
* @param resourceManagerLeaderID current leader id of the ResourceManager
* @return answer to the slot request
*/
- Future<SlotRequestReply> requestSlot(
+ Future<TMSlotRequestReply> requestSlot(
+ SlotID slotID,
AllocationID allocationID,
UUID resourceManagerLeaderID,
@RpcTimeout Time timeout);
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java
index b4b3bae..2dbd9eb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java
@@ -113,7 +113,7 @@ public class TaskExecutorToResourceManagerConnection
ResourceManagerGateway resourceManager, UUID leaderId, long timeoutMillis) throws Exception {
Time timeout = Time.milliseconds(timeoutMillis);
- return resourceManager.registerTaskExecutor(leaderId, taskExecutorAddress, resourceID, timeout);
+ return resourceManager.registerTaskExecutor(leaderId, taskExecutorAddress, resourceID, new SlotReport(), timeout);
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
index fdb83f5..ce1fdca 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.resourcemanager;
import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
import org.apache.flink.runtime.rpc.MainThreadExecutable;
import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcGateway;
@@ -43,7 +44,8 @@ public class ResourceManagerHATest {
@Test
public void testGrantAndRevokeLeadership() throws Exception {
- // mock a RpcService which will return a special RpcGateway when call its startServer method, the returned RpcGateway directly execute runAsync call
+ // mock a RpcService which will return a special RpcGateway when call its startServer method,
+ // the returned RpcGateway directly executes runAsync calls
TestingResourceManagerGatewayProxy gateway = mock(TestingResourceManagerGatewayProxy.class);
doCallRealMethod().when(gateway).runAsync(any(Runnable.class));
@@ -54,18 +56,18 @@ public class ResourceManagerHATest {
TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
highAvailabilityServices.setResourceManagerLeaderElectionService(leaderElectionService);
- SlotManager slotManager = mock(SlotManager.class);
- final ResourceManager resourceManager = new StandaloneResourceManager(rpcService, highAvailabilityServices, slotManager);
+ final ResourceManager resourceManager =
+ new TestingResourceManager(rpcService, highAvailabilityServices);
resourceManager.start();
// before grant leadership, resourceManager's leaderId is null
- Assert.assertNull(resourceManager.getLeaderSessionID());
+ Assert.assertEquals(new UUID(0,0), resourceManager.getLeaderSessionID());
final UUID leaderId = UUID.randomUUID();
leaderElectionService.isLeader(leaderId);
// after grant leadership, resourceManager's leaderId has value
Assert.assertEquals(leaderId, resourceManager.getLeaderSessionID());
// then revoke leadership, resourceManager's leaderId is null again
leaderElectionService.notLeader();
- Assert.assertNull(resourceManager.getLeaderSessionID());
+ Assert.assertEquals(new UUID(0,0), resourceManager.getLeaderSessionID());
}
private static abstract class TestingResourceManagerGatewayProxy implements MainThreadExecutable, StartStoppable, RpcGateway {
[50/50] [abbrv] flink git commit: [FLINK-4871] [mini cluster] Add
memory calculation for TaskManagers to MiniCluster
Posted by tr...@apache.org.
[FLINK-4871] [mini cluster] Add memory calculation for TaskManagers to MiniCluster
If the managed memory size for the task manager has not been set in the Configuration, then
it is automatically calculated by dividing the available memory by the number of distributed
components. Additionally this PR allows to provide a MetricRegistry to the TaskManagerRunner.
That way it is possible to use the MiniCluster's MetricRegistry.
Add memory calculation for task managers
This closes #2669.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/09ef605c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/09ef605c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/09ef605c
Branch: refs/heads/flip-6
Commit: 09ef605ce59a0dcce07adced8e4ea9c7ba3bc647
Parents: 2a5a5b3
Author: Till Rohrmann <tr...@apache.org>
Authored: Thu Oct 20 11:07:08 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:35 2016 +0100
----------------------------------------------------------------------
.../flink/runtime/minicluster/MiniCluster.java | 7 +-
.../minicluster/MiniClusterConfiguration.java | 86 ++++++++++++++-
.../resourcemanager/JobLeaderIdService.java | 2 +-
.../resourcemanager/ResourceManager.java | 3 +-
.../resourcemanager/ResourceManagerRunner.java | 102 ++++++++++++++++++
.../exceptions/ConfigurationException.java | 2 +-
.../exceptions/ResourceManagerRunner.java | 105 -------------------
.../runtime/taskexecutor/TaskExecutor.java | 41 +++-----
.../runtime/taskexecutor/TaskManagerRunner.java | 28 ++++-
.../taskexecutor/TaskManagerServices.java | 28 -----
.../TaskManagerServicesConfiguration.java | 19 +---
.../minicluster/LocalFlinkMiniCluster.scala | 4 +-
.../flink/runtime/taskmanager/TaskManager.scala | 5 +-
13 files changed, 236 insertions(+), 196 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/09ef605c/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
index b005330..611d4c4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
@@ -32,7 +32,7 @@ import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
-import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerRunner;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerRunner;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
import org.apache.flink.runtime.taskexecutor.TaskManagerRunner;
@@ -161,7 +161,7 @@ public class MiniCluster {
LOG.info("Starting Flink Mini Cluster");
LOG.debug("Using configuration {}", config);
- final Configuration configuration = new UnmodifiableConfiguration(config.getConfiguration());
+ final Configuration configuration = new UnmodifiableConfiguration(config.generateConfiguration());
final Time rpcTimeout = config.getRpcTimeout();
final int numJobManagers = config.getNumJobManagers();
final int numTaskManagers = config.getNumTaskManagers();
@@ -468,7 +468,8 @@ public class MiniCluster {
configuration,
new ResourceID(UUID.randomUUID().toString()),
taskManagerRpcServices[i],
- haServices);
+ haServices,
+ metricRegistry);
taskManagerRunners[i].start();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/09ef605c/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java
index cfbbffb..3a03ca3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java
@@ -20,8 +20,11 @@ package org.apache.flink.runtime.minicluster;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.util.EnvironmentInformation;
import scala.concurrent.duration.FiniteDuration;
import static org.apache.flink.util.Preconditions.checkArgument;
@@ -41,6 +44,8 @@ public class MiniClusterConfiguration {
private String commonBindAddress;
+ private long managedMemoryPerTaskManager = -1;
+
// ------------------------------------------------------------------------
// Construction
// ------------------------------------------------------------------------
@@ -96,14 +101,15 @@ public class MiniClusterConfiguration {
this.commonBindAddress = bindAddress;
}
+ public void setManagedMemoryPerTaskManager(long managedMemoryPerTaskManager) {
+ checkArgument(managedMemoryPerTaskManager > 0, "must have more than 0 MB of memory for the TaskManager.");
+ this.managedMemoryPerTaskManager = managedMemoryPerTaskManager;
+ }
+
// ------------------------------------------------------------------------
// getters
// ------------------------------------------------------------------------
- public Configuration getConfiguration() {
- return config;
- }
-
public boolean getUseSingleRpcSystem() {
return singleRpcService;
}
@@ -147,10 +153,23 @@ public class MiniClusterConfiguration {
return Time.of(duration.length(), duration.unit());
}
+ public long getManagedMemoryPerTaskManager() {
+ return getOrCalculateManagedMemoryPerTaskManager();
+ }
+
// ------------------------------------------------------------------------
// utils
// ------------------------------------------------------------------------
+ public Configuration generateConfiguration() {
+ Configuration newConfiguration = new Configuration(config);
+ // set the memory
+ long memory = getOrCalculateManagedMemoryPerTaskManager();
+ newConfiguration.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, memory);
+
+ return newConfiguration;
+ }
+
@Override
public String toString() {
return "MiniClusterConfiguration {" +
@@ -162,4 +181,63 @@ public class MiniClusterConfiguration {
", config=" + config +
'}';
}
+
+ /**
+ * Get or calculate the managed memory per task manager. The memory is calculated in the
+ * following order:
+ *
+ * 1. Return {@link #managedMemoryPerTaskManager} if set
+ * 2. Return config.getInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY) if set
+ * 3. Distribute the available free memory equally among all components (JMs, RMs and TMs) and
+ * calculate the managed memory from the share of memory for a single task manager.
+ *
+ * @return
+ */
+ private long getOrCalculateManagedMemoryPerTaskManager() {
+ if (managedMemoryPerTaskManager == -1) {
+ // no memory set in the mini cluster configuration
+ final ConfigOption<Integer> memorySizeOption = ConfigOptions
+ .key(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY)
+ .defaultValue(-1);
+
+ int memorySize = config.getInteger(memorySizeOption);
+
+ if (memorySize == -1) {
+ // no memory set in the flink configuration
+ // share the available memory among all running components
+ final ConfigOption<Integer> bufferSizeOption = ConfigOptions
+ .key(ConfigConstants.TASK_MANAGER_MEMORY_SEGMENT_SIZE_KEY)
+ .defaultValue(ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_SEGMENT_SIZE);
+
+ final ConfigOption<Long> bufferMemoryOption = ConfigOptions
+ .key(ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY)
+ .defaultValue((long) ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS);
+
+ final ConfigOption<Float> memoryFractionOption = ConfigOptions
+ .key(ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY)
+ .defaultValue(ConfigConstants.DEFAULT_MEMORY_MANAGER_MEMORY_FRACTION);
+
+ float memoryFraction = config.getFloat(memoryFractionOption);
+ long networkBuffersMemory = config.getLong(bufferMemoryOption) * config.getInteger(bufferSizeOption);
+
+ long freeMemory = EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag();
+
+ // we assign each component the same amount of free memory
+ // (might be a bit of overkill for the JMs and RMs)
+ long memoryPerComponent = freeMemory / (numTaskManagers + numResourceManagers + numJobManagers);
+
+ // subtract the network buffer memory
+ long memoryMinusNetworkBuffers = memoryPerComponent - networkBuffersMemory;
+
+ // calculate the managed memory size
+ long managedMemoryBytes = (long) (memoryMinusNetworkBuffers * memoryFraction);
+
+ return managedMemoryBytes >>> 20;
+ } else {
+ return memorySize;
+ }
+ } else {
+ return managedMemoryPerTaskManager;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/09ef605c/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdService.java
index 6c7e249..56e72c0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdService.java
@@ -59,7 +59,7 @@ public class JobLeaderIdService {
/** Actions to call when the job leader changes */
private JobLeaderIdActions jobLeaderIdActions;
- public JobLeaderIdService(HighAvailabilityServices highAvailabilityServices) throws Exception {
+ public JobLeaderIdService(HighAvailabilityServices highAvailabilityServices) {
this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices);
this.runningJobsRegistry = highAvailabilityServices.getRunningJobsRegistry();
http://git-wip-us.apache.org/repos/asf/flink/blob/09ef605c/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index 7240087..a81c214 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -156,8 +156,9 @@ public abstract class ResourceManager<WorkerType extends Serializable>
throw new ResourceManagerException("Could not create the slot manager.", e);
}
+ leaderElectionService = highAvailabilityServices.getResourceManagerLeaderElectionService();
+
try {
- leaderElectionService = highAvailabilityServices.getResourceManagerLeaderElectionService();
leaderElectionService.start(this);
} catch (Exception e) {
throw new ResourceManagerException("Could not start the leader election service.", e);
http://git-wip-us.apache.org/repos/asf/flink/blob/09ef605c/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
new file mode 100644
index 0000000..959b727
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.resourcemanager.exceptions.ConfigurationException;
+import org.apache.flink.runtime.resourcemanager.slotmanager.DefaultSlotManager;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
+import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Simple {@link StandaloneResourceManager} runner. It instantiates the resource manager's services
+ * and handles fatal errors by shutting the resource manager down.
+ */
+public class ResourceManagerRunner implements FatalErrorHandler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ResourceManagerRunner.class);
+
+ private final Object lock = new Object();
+
+ private final ResourceManager<?> resourceManager;
+
+ public ResourceManagerRunner(
+ final Configuration configuration,
+ final RpcService rpcService,
+ final HighAvailabilityServices highAvailabilityServices,
+ final MetricRegistry metricRegistry) throws ConfigurationException {
+
+ Preconditions.checkNotNull(configuration);
+ Preconditions.checkNotNull(rpcService);
+ Preconditions.checkNotNull(highAvailabilityServices);
+ Preconditions.checkNotNull(metricRegistry);
+
+ final ResourceManagerConfiguration resourceManagerConfiguration = ResourceManagerConfiguration.fromConfiguration(configuration);
+ final SlotManagerFactory slotManagerFactory = new DefaultSlotManager.Factory();
+ final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(highAvailabilityServices);
+
+ this.resourceManager = new StandaloneResourceManager(
+ rpcService,
+ resourceManagerConfiguration,
+ highAvailabilityServices,
+ slotManagerFactory,
+ metricRegistry,
+ jobLeaderIdService,
+ this);
+ }
+
+ //-------------------------------------------------------------------------------------
+ // Lifecycle management
+ //-------------------------------------------------------------------------------------
+
+ public void start() throws Exception {
+ resourceManager.start();
+ }
+
+ public void shutDown() throws Exception {
+ shutDownInternally();
+ }
+
+ private void shutDownInternally() throws Exception {
+ synchronized (lock) {
+ resourceManager.shutDown();
+ }
+ }
+
+ //-------------------------------------------------------------------------------------
+ // Fatal error handler
+ //-------------------------------------------------------------------------------------
+
+ @Override
+ public void onFatalError(Throwable exception) {
+ LOG.error("Encountered fatal error.", exception);
+
+ try {
+ shutDownInternally();
+ } catch (Exception e) {
+ LOG.error("Could not properly shut down the resource manager.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/09ef605c/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ConfigurationException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ConfigurationException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ConfigurationException.java
index f081fff..0007318 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ConfigurationException.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ConfigurationException.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.resourcemanager.exceptions;
/**
- * Base class for configuration related exception which occur when creating a configuration.
+ * Exception which occures when creating a configuration object fails.
*/
public class ConfigurationException extends Exception {
private static final long serialVersionUID = 3971647332059381556L;
http://git-wip-us.apache.org/repos/asf/flink/blob/09ef605c/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerRunner.java
deleted file mode 100644
index 0c7e4e4..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerRunner.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.resourcemanager.exceptions;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.metrics.MetricRegistry;
-import org.apache.flink.runtime.resourcemanager.JobLeaderIdService;
-import org.apache.flink.runtime.resourcemanager.ResourceManager;
-import org.apache.flink.runtime.resourcemanager.ResourceManagerConfiguration;
-import org.apache.flink.runtime.resourcemanager.StandaloneResourceManager;
-import org.apache.flink.runtime.resourcemanager.slotmanager.DefaultSlotManager;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
-import org.apache.flink.runtime.rpc.FatalErrorHandler;
-import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.util.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Simple {@link StandaloneResourceManager} runner. It instantiates the resource manager's services
- * and handles fatal errors by shutting the resource manager down.
- */
-public class ResourceManagerRunner implements FatalErrorHandler {
-
- private static final Logger LOG = LoggerFactory.getLogger(ResourceManagerRunner.class);
-
- private final Object lock = new Object();
-
- private final ResourceManager<?> resourceManager;
-
- public ResourceManagerRunner(
- final Configuration configuration,
- final RpcService rpcService,
- final HighAvailabilityServices highAvailabilityServices,
- final MetricRegistry metricRegistry) throws Exception {
-
- Preconditions.checkNotNull(configuration);
- Preconditions.checkNotNull(rpcService);
- Preconditions.checkNotNull(highAvailabilityServices);
- Preconditions.checkNotNull(metricRegistry);
-
- final ResourceManagerConfiguration resourceManagerConfiguration = ResourceManagerConfiguration.fromConfiguration(configuration);
- final SlotManagerFactory slotManagerFactory = new DefaultSlotManager.Factory();
- final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(highAvailabilityServices);
-
- this.resourceManager = new StandaloneResourceManager(
- rpcService,
- resourceManagerConfiguration,
- highAvailabilityServices,
- slotManagerFactory,
- metricRegistry,
- jobLeaderIdService,
- this);
- }
-
- //-------------------------------------------------------------------------------------
- // Lifecycle management
- //-------------------------------------------------------------------------------------
-
- public void start() throws Exception {
- resourceManager.start();
- }
-
- public void shutDown() throws Exception {
- shutDownInternally();
- }
-
- private void shutDownInternally() throws Exception {
- synchronized (lock) {
- resourceManager.shutDown();
- }
- }
-
- //-------------------------------------------------------------------------------------
- // Fatal error handler
- //-------------------------------------------------------------------------------------
-
- @Override
- public void onFatalError(Throwable exception) {
- LOG.error("Encountered fatal error.", exception);
-
- try {
- shutDownInternally();
- } catch (Exception e) {
- LOG.error("Could not properly shut down the resource manager.", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/09ef605c/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index aea926c..5eb8e6a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -78,6 +78,7 @@ import org.apache.flink.runtime.taskmanager.Task;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import org.apache.flink.runtime.taskmanager.TaskManagerActions;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
+import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.Preconditions;
import java.io.IOException;
@@ -215,47 +216,33 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
* Called to shut down the TaskManager. The method closes all TaskManager services.
*/
@Override
- public void shutDown() {
+ public void shutDown() throws Exception {
log.info("Stopping TaskManager {}.", getAddress());
+ Exception exception = null;
+
taskSlotTable.stop();
if (isConnectedToResourceManager()) {
- try {
- resourceManagerConnection.close();
- } catch (Exception e) {
- log.error("Could not cleanly close the ResourceManager connection.", e);
- }
+ resourceManagerConnection.close();
}
- try {
- ioManager.shutdown();
- } catch (Exception e) {
- log.error("IOManager did not shut down properly.", e);
- }
+ ioManager.shutdown();
- try {
- memoryManager.shutdown();
- } catch (Exception e) {
- log.error("MemoryManager did not shut down properly.", e);
- }
+ memoryManager.shutdown();
- try {
- networkEnvironment.shutdown();
- } catch (Exception e) {
- log.error("Network environment did not shut down properly.", e);
- }
+ networkEnvironment.shutdown();
+
+ fileCache.shutdown();
try {
- fileCache.shutdown();
+ super.shutDown();
} catch (Exception e) {
- log.error("File cache did not shut down properly.", e);
+ exception = ExceptionUtils.firstOrSuppressed(e, exception);
}
- try {
- metricRegistry.shutdown();
- } catch (Exception e) {
- log.error("MetricRegistry did not shut down properly.", e);
+ if (exception != null) {
+ ExceptionUtils.rethrowException(exception, "Error while shutting the TaskExecutor down.");
}
log.info("Stopped TaskManager {}.", getAddress());
http://git-wip-us.apache.org/repos/asf/flink/blob/09ef605c/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
index 7d9ee55..99a7c5d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
@@ -24,9 +24,12 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.rpc.RpcServiceUtils;
+import org.apache.flink.runtime.taskexecutor.utils.TaskExecutorMetricsInitializer;
import org.apache.flink.runtime.util.LeaderRetrievalUtils;
import org.apache.flink.util.Preconditions;
@@ -66,7 +69,8 @@ public class TaskManagerRunner implements FatalErrorHandler {
Configuration configuration,
ResourceID resourceID,
RpcService rpcService,
- HighAvailabilityServices highAvailabilityServices) throws Exception {
+ HighAvailabilityServices highAvailabilityServices,
+ MetricRegistry metricRegistry) throws Exception {
this.configuration = Preconditions.checkNotNull(configuration);
this.resourceID = Preconditions.checkNotNull(resourceID);
@@ -81,10 +85,20 @@ public class TaskManagerRunner implements FatalErrorHandler {
remoteAddress,
false);
- TaskManagerServices taskManagerServices = TaskManagerServices.fromConfiguration(taskManagerServicesConfiguration, resourceID);
+ TaskManagerServices taskManagerServices = TaskManagerServices.fromConfiguration(
+ taskManagerServicesConfiguration,
+ resourceID);
TaskManagerConfiguration taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration);
+ TaskManagerMetricGroup taskManagerMetricGroup = new TaskManagerMetricGroup(
+ metricRegistry,
+ taskManagerServices.getTaskManagerLocation().getHostname(),
+ resourceID.toString());
+
+ // Initialize the TM metrics
+ TaskExecutorMetricsInitializer.instantiateStatusMetrics(taskManagerMetricGroup, taskManagerServices.getNetworkEnvironment());
+
this.taskManager = new TaskExecutor(
taskManagerConfiguration,
taskManagerServices.getTaskManagerLocation(),
@@ -93,8 +107,8 @@ public class TaskManagerRunner implements FatalErrorHandler {
taskManagerServices.getIOManager(),
taskManagerServices.getNetworkEnvironment(),
highAvailabilityServices,
- taskManagerServices.getMetricRegistry(),
- taskManagerServices.getTaskManagerMetricGroup(),
+ metricRegistry,
+ taskManagerMetricGroup,
taskManagerServices.getBroadcastVariableManager(),
taskManagerServices.getFileCache(),
taskManagerServices.getTaskSlotTable(),
@@ -117,7 +131,11 @@ public class TaskManagerRunner implements FatalErrorHandler {
protected void shutDownInternally() {
synchronized(lock) {
- taskManager.shutDown();
+ try {
+ taskManager.shutDown();
+ } catch (Exception e) {
+ LOG.error("Could not properly shut down the task manager.", e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/09ef605c/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
index e8de1b1..7966078 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
@@ -36,13 +36,11 @@ import org.apache.flink.runtime.io.network.netty.NettyConnectionManager;
import org.apache.flink.runtime.io.network.partition.ResultPartitionManager;
import org.apache.flink.runtime.memory.MemoryManager;
import org.apache.flink.runtime.metrics.MetricRegistry;
-import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
import org.apache.flink.runtime.query.KvStateRegistry;
import org.apache.flink.runtime.query.netty.DisabledKvStateRequestStats;
import org.apache.flink.runtime.query.netty.KvStateServer;
import org.apache.flink.runtime.taskexecutor.slot.TaskSlotTable;
import org.apache.flink.runtime.taskexecutor.slot.TimerService;
-import org.apache.flink.runtime.taskexecutor.utils.TaskExecutorMetricsInitializer;
import org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.runtime.util.EnvironmentInformation;
@@ -68,8 +66,6 @@ public class TaskManagerServices {
private final MemoryManager memoryManager;
private final IOManager ioManager;
private final NetworkEnvironment networkEnvironment;
- private final MetricRegistry metricRegistry;
- private final TaskManagerMetricGroup taskManagerMetricGroup;
private final BroadcastVariableManager broadcastVariableManager;
private final FileCache fileCache;
private final TaskSlotTable taskSlotTable;
@@ -81,8 +77,6 @@ public class TaskManagerServices {
MemoryManager memoryManager,
IOManager ioManager,
NetworkEnvironment networkEnvironment,
- MetricRegistry metricRegistry,
- TaskManagerMetricGroup taskManagerMetricGroup,
BroadcastVariableManager broadcastVariableManager,
FileCache fileCache,
TaskSlotTable taskSlotTable,
@@ -93,8 +87,6 @@ public class TaskManagerServices {
this.memoryManager = Preconditions.checkNotNull(memoryManager);
this.ioManager = Preconditions.checkNotNull(ioManager);
this.networkEnvironment = Preconditions.checkNotNull(networkEnvironment);
- this.metricRegistry = Preconditions.checkNotNull(metricRegistry);
- this.taskManagerMetricGroup = Preconditions.checkNotNull(taskManagerMetricGroup);
this.broadcastVariableManager = Preconditions.checkNotNull(broadcastVariableManager);
this.fileCache = Preconditions.checkNotNull(fileCache);
this.taskSlotTable = Preconditions.checkNotNull(taskSlotTable);
@@ -122,14 +114,6 @@ public class TaskManagerServices {
return taskManagerLocation;
}
- public MetricRegistry getMetricRegistry() {
- return metricRegistry;
- }
-
- public TaskManagerMetricGroup getTaskManagerMetricGroup() {
- return taskManagerMetricGroup;
- }
-
public BroadcastVariableManager getBroadcastVariableManager() {
return broadcastVariableManager;
}
@@ -181,16 +165,6 @@ public class TaskManagerServices {
// start the I/O manager, it will create some temp directories.
final IOManager ioManager = new IOManagerAsync(taskManagerServicesConfiguration.getTmpDirPaths());
- final MetricRegistry metricRegistry = new MetricRegistry(taskManagerServicesConfiguration.getMetricRegistryConfiguration());
-
- final TaskManagerMetricGroup taskManagerMetricGroup = new TaskManagerMetricGroup(
- metricRegistry,
- taskManagerLocation.getHostname(),
- taskManagerLocation.getResourceID().toString());
-
- // Initialize the TM metrics
- TaskExecutorMetricsInitializer.instantiateStatusMetrics(taskManagerMetricGroup, network);
-
final BroadcastVariableManager broadcastVariableManager = new BroadcastVariableManager();
final FileCache fileCache = new FileCache(taskManagerServicesConfiguration.getTmpDirPaths());
@@ -214,8 +188,6 @@ public class TaskManagerServices {
memoryManager,
ioManager,
network,
- metricRegistry,
- taskManagerMetricGroup,
broadcastVariableManager,
fileCache,
taskSlotTable,
http://git-wip-us.apache.org/repos/asf/flink/blob/09ef605c/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
index 036a890..702142f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
@@ -28,7 +28,6 @@ import org.apache.flink.core.memory.MemoryType;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.network.netty.NettyConfig;
import org.apache.flink.runtime.memory.MemoryManager;
-import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
import org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration;
import org.apache.flink.util.MathUtils;
@@ -58,8 +57,6 @@ public class TaskManagerServicesConfiguration {
private final float memoryFraction;
- private final MetricRegistryConfiguration metricRegistryConfiguration;
-
public TaskManagerServicesConfiguration(
InetAddress taskManagerAddress,
String[] tmpDirPaths,
@@ -67,8 +64,7 @@ public class TaskManagerServicesConfiguration {
int numberOfSlots,
long configuredMemory,
boolean preAllocateMemory,
- float memoryFraction,
- MetricRegistryConfiguration metricRegistryConfiguration) {
+ float memoryFraction) {
this.taskManagerAddress = checkNotNull(taskManagerAddress);
this.tmpDirPaths = checkNotNull(tmpDirPaths);
@@ -78,8 +74,6 @@ public class TaskManagerServicesConfiguration {
this.configuredMemory = configuredMemory;
this.preAllocateMemory = preAllocateMemory;
this.memoryFraction = memoryFraction;
-
- this.metricRegistryConfiguration = checkNotNull(metricRegistryConfiguration);
}
// --------------------------------------------------------------------------------------------
@@ -113,10 +107,6 @@ public class TaskManagerServicesConfiguration {
return preAllocateMemory;
}
- public MetricRegistryConfiguration getMetricRegistryConfiguration() {
- return metricRegistryConfiguration;
- }
-
// --------------------------------------------------------------------------------------------
// Parsing of Flink configuration
// --------------------------------------------------------------------------------------------
@@ -171,10 +161,6 @@ public class TaskManagerServicesConfiguration {
ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY,
"MemoryManager fraction of the free memory must be between 0.0 and 1.0");
- final MetricRegistryConfiguration metricRegistryConfiguration = MetricRegistryConfiguration.fromConfiguration(configuration);
-
-
-
return new TaskManagerServicesConfiguration(
remoteAddress,
tmpDirs,
@@ -182,8 +168,7 @@ public class TaskManagerServicesConfiguration {
slots,
configuredMemory,
preAllocateMemory,
- memoryFraction,
- metricRegistryConfiguration);
+ memoryFraction);
}
// --------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/09ef605c/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
index eac0a51..3a77283 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
@@ -42,7 +42,7 @@ import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService
import org.apache.flink.runtime.memory.MemoryManager
import org.apache.flink.runtime.messages.JobManagerMessages
import org.apache.flink.runtime.messages.JobManagerMessages.{RunningJobsStatus, StoppingFailure, StoppingResponse}
-import org.apache.flink.runtime.metrics.MetricRegistry
+import org.apache.flink.runtime.metrics.{MetricRegistry, MetricRegistryConfiguration}
import org.apache.flink.runtime.taskexecutor.{TaskManagerConfiguration, TaskManagerServices, TaskManagerServicesConfiguration}
import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerLocation}
import org.apache.flink.runtime.util.{EnvironmentInformation, LeaderRetrievalUtils}
@@ -209,7 +209,7 @@ class LocalFlinkMiniCluster(
taskManagerServicesConfiguration,
resourceID)
- val metricRegistry = taskManagerServices.getMetricRegistry()
+ val metricRegistry = new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config))
val props = getTaskManagerProps(
taskManagerClass,
http://git-wip-us.apache.org/repos/asf/flink/blob/09ef605c/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 5e0510d..5ecfb80 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -56,7 +56,7 @@ import org.apache.flink.runtime.messages.StackTraceSampleMessages.{SampleTaskSta
import org.apache.flink.runtime.messages.TaskManagerMessages._
import org.apache.flink.runtime.messages.TaskMessages._
import org.apache.flink.runtime.messages.checkpoint.{AbstractCheckpointMessage, NotifyCheckpointComplete, TriggerCheckpoint}
-import org.apache.flink.runtime.metrics.{MetricRegistry => FlinkMetricRegistry}
+import org.apache.flink.runtime.metrics.{MetricRegistryConfiguration, MetricRegistry => FlinkMetricRegistry}
import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup
import org.apache.flink.runtime.metrics.util.MetricUtils
import org.apache.flink.runtime.process.ProcessReaper
@@ -1842,7 +1842,8 @@ object TaskManager {
taskManagerServicesConfiguration,
resourceID)
- val metricRegistry = taskManagerServices.getMetricRegistry()
+ val metricRegistry = new FlinkMetricRegistry(
+ MetricRegistryConfiguration.fromConfiguration(configuration))
val leaderRetrievalService = leaderRetrievalServiceOption match {
case Some(lrs) => lrs
[05/50] [abbrv] flink git commit: [FLINK-4580] [rpc] Report rpc
invocation exceptions to the caller
Posted by tr...@apache.org.
[FLINK-4580] [rpc] Report rpc invocation exceptions to the caller
This closes #2526.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4f331858
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4f331858
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4f331858
Branch: refs/heads/flip-6
Commit: 4f3318588ba598aad28914db836127e587b8c9bb
Parents: 44422f8
Author: Till Rohrmann <tr...@apache.org>
Authored: Wed Sep 21 15:18:27 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:27 2016 +0100
----------------------------------------------------------------------
.../flink/runtime/rpc/akka/AkkaRpcActor.java | 53 ++++++++++++--------
.../runtime/rpc/akka/AkkaRpcActorTest.java | 34 +++++++++++++
2 files changed, 66 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/4f331858/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java
index 59daa46..1b456a7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java
@@ -35,6 +35,7 @@ import org.apache.flink.runtime.rpc.akka.messages.Processing;
import org.apache.flink.runtime.rpc.akka.messages.RpcInvocation;
import org.apache.flink.runtime.rpc.akka.messages.RunAsync;
+import org.apache.flink.runtime.rpc.exceptions.RpcConnectionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -86,11 +87,11 @@ class AkkaRpcActor<C extends RpcGateway, T extends RpcEndpoint<C>> extends Untyp
unstashAll();
getContext().become(new Procedure<Object>() {
@Override
- public void apply(Object message) throws Exception {
- if (message.equals(Processing.STOP)) {
+ public void apply(Object msg) throws Exception {
+ if (msg.equals(Processing.STOP)) {
getContext().unbecome();
} else {
- handleMessage(message);
+ handleMessage(msg);
}
}
});
@@ -130,21 +131,36 @@ class AkkaRpcActor<C extends RpcGateway, T extends RpcEndpoint<C>> extends Untyp
* @param rpcInvocation Rpc invocation message
*/
private void handleRpcInvocation(RpcInvocation rpcInvocation) {
+ Method rpcMethod = null;
+
try {
String methodName = rpcInvocation.getMethodName();
Class<?>[] parameterTypes = rpcInvocation.getParameterTypes();
- Method rpcMethod = lookupRpcMethod(methodName, parameterTypes);
+ rpcMethod = lookupRpcMethod(methodName, parameterTypes);
+ } catch(ClassNotFoundException e) {
+ LOG.error("Could not load method arguments.", e);
+
+ RpcConnectionException rpcException = new RpcConnectionException("Could not load method arguments.", e);
+ getSender().tell(new Status.Failure(rpcException), getSelf());
+ } catch (IOException e) {
+ LOG.error("Could not deserialize rpc invocation message.", e);
+
+ RpcConnectionException rpcException = new RpcConnectionException("Could not deserialize rpc invocation message.", e);
+ getSender().tell(new Status.Failure(rpcException), getSelf());
+ } catch (final NoSuchMethodException e) {
+ LOG.error("Could not find rpc method for rpc invocation.", e);
+
+ RpcConnectionException rpcException = new RpcConnectionException("Could not find rpc method for rpc invocation.", e);
+ getSender().tell(new Status.Failure(rpcException), getSelf());
+ }
- if (rpcMethod.getReturnType().equals(Void.TYPE)) {
- // No return value to send back
- try {
+ if (rpcMethod != null) {
+ try {
+ if (rpcMethod.getReturnType().equals(Void.TYPE)) {
+ // No return value to send back
rpcMethod.invoke(rpcEndpoint, rpcInvocation.getArgs());
- } catch (Throwable e) {
- LOG.error("Error while executing remote procedure call {}.", rpcMethod, e);
- }
- } else {
- try {
+ } else {
Object result = rpcMethod.invoke(rpcEndpoint, rpcInvocation.getArgs());
if (result instanceof Future) {
@@ -169,17 +185,12 @@ class AkkaRpcActor<C extends RpcGateway, T extends RpcEndpoint<C>> extends Untyp
// tell the sender the result of the computation
getSender().tell(new Status.Success(result), getSelf());
}
- } catch (Throwable e) {
- // tell the sender about the failure
- getSender().tell(new Status.Failure(e), getSelf());
}
+ } catch (Throwable e) {
+ LOG.error("Error while executing remote procedure call {}.", rpcMethod, e);
+ // tell the sender about the failure
+ getSender().tell(new Status.Failure(e), getSelf());
}
- } catch(ClassNotFoundException e) {
- LOG.error("Could not load method arguments.", e);
- } catch (IOException e) {
- LOG.error("Could not deserialize rpc invocation message.", e);
- } catch (final NoSuchMethodException e) {
- LOG.error("Could not find rpc method for rpc invocation: {}.", rpcInvocation, e);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/4f331858/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
index 5624d12..1e8c9a6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java
@@ -119,10 +119,44 @@ public class AkkaRpcActorTest extends TestLogger {
rpcEndpoint.shutDown();
}
+ /**
+ * Tests that we receive a RpcConnectionException when calling a rpc method (with return type)
+ * on a wrong rpc endpoint.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testWrongGatewayEndpointConnection() throws Exception {
+ DummyRpcEndpoint rpcEndpoint = new DummyRpcEndpoint(akkaRpcService);
+
+ rpcEndpoint.start();
+
+ Future<WrongRpcGateway> futureGateway = akkaRpcService.connect(rpcEndpoint.getAddress(), WrongRpcGateway.class);
+
+ WrongRpcGateway gateway = Await.result(futureGateway, timeout.duration());
+
+ // since it is a tell operation we won't receive a RpcConnectionException, it's only logged
+ gateway.tell("foobar");
+
+ Future<Boolean> result = gateway.barfoo();
+
+ try {
+ Await.result(result, timeout.duration());
+ fail("We expected a RpcConnectionException.");
+ } catch (RpcConnectionException rpcConnectionException) {
+ // we expect this exception here
+ }
+ }
+
private interface DummyRpcGateway extends RpcGateway {
Future<Integer> foobar();
}
+ private interface WrongRpcGateway extends RpcGateway {
+ Future<Boolean> barfoo();
+ void tell(String message);
+ }
+
private static class DummyRpcEndpoint extends RpcEndpoint<DummyRpcGateway> {
private volatile int _foobar = 42;
[40/50] [abbrv] flink git commit: [hotfix] [cluster management]
Remove scala dependencies from MiniCluster.java
Posted by tr...@apache.org.
[hotfix] [cluster management] Remove scala dependencies from MiniCluster.java
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/73dafd4a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/73dafd4a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/73dafd4a
Branch: refs/heads/flip-6
Commit: 73dafd4a7145a85c9f80e15c7d783ada73e7b8a4
Parents: 8ab27c0
Author: Till Rohrmann <tr...@apache.org>
Authored: Mon Oct 17 14:41:52 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:33 2016 +0100
----------------------------------------------------------------------
.../flink/runtime/minicluster/MiniCluster.java | 12 ++++--------
.../org/apache/flink/runtime/akka/AkkaUtils.scala | 17 +++++++++++++++++
.../runtime/minicluster/MiniClusterITCase.java | 2 +-
3 files changed, 22 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/73dafd4a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
index 1ee38e0..d85234d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
@@ -35,9 +35,6 @@ import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
import org.apache.flink.util.ExceptionUtils;
-import scala.Option;
-import scala.Tuple2;
-
import javax.annotation.concurrent.GuardedBy;
import static org.apache.flink.util.ExceptionUtils.firstOrSuppressed;
@@ -348,7 +345,7 @@ public class MiniCluster {
/**
* Factory method to instantiate the RPC service.
*
- * @param config
+ * @param configuration
* The configuration of the mini cluster
* @param askTimeout
* The default RPC timeout for asynchronous "ask" requests.
@@ -360,17 +357,16 @@ public class MiniCluster {
* @return The instantiated RPC service
*/
protected RpcService createRpcService(
- Configuration config,
+ Configuration configuration,
Time askTimeout,
boolean remoteEnabled,
String bindAddress) {
ActorSystem actorSystem;
if (remoteEnabled) {
- Tuple2<String, Object> remoteSettings = new Tuple2<String, Object>(bindAddress, 0);
- actorSystem = AkkaUtils.createActorSystem(config, Option.apply(remoteSettings));
+ actorSystem = AkkaUtils.createActorSystem(configuration, bindAddress, 0);
} else {
- actorSystem = AkkaUtils.createLocalActorSystem(config);
+ actorSystem = AkkaUtils.createLocalActorSystem(configuration);
}
return new AkkaRpcService(actorSystem, askTimeout);
http://git-wip-us.apache.org/repos/asf/flink/blob/73dafd4a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
index 1f23779..9ac9773 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
@@ -58,6 +58,23 @@ object AkkaUtils {
}
/**
+ * Creates an actor system bound to the given hostname and port.
+ *
+ * @param configuration instance containing the user provided configuration values
+ * @param hostname of the network interface to bind to
+ * @param port of to bind to
+ * @return created actor system
+ */
+ def createActorSystem(
+ configuration: Configuration,
+ hostname: String,
+ port: Int)
+ : ActorSystem = {
+
+ createActorSystem(configuration, Some((hostname, port)))
+ }
+
+ /**
* Creates an actor system. If a listening address is specified, then the actor system will listen
* on that address for messages from a remote actor system. If not, then a local actor system
* will be instantiated.
http://git-wip-us.apache.org/repos/asf/flink/blob/73dafd4a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
index dd43337..ef53547 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
@@ -31,7 +31,7 @@ import org.junit.Test;
*/
public class MiniClusterITCase extends TestLogger {
-// @Test
+ @Test
public void runJobWithSingleRpcService() throws Exception {
MiniClusterConfiguration cfg = new MiniClusterConfiguration();
[18/50] [abbrv] flink git commit: [FLINK-4406] [cluster management]
Implement job master registration at resource manager
Posted by tr...@apache.org.
[FLINK-4406] [cluster management] Implement job master registration at resource manager
[FLINK-4406] [cluster management] Skip new connection if new resource manager's address and leader id are both not changing
[FLINK-4406] [cluster management] Verify registration response with leader id
This closes #2565.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/dd8c9b9e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/dd8c9b9e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/dd8c9b9e
Branch: refs/heads/flip-6
Commit: dd8c9b9e914b91f9f9890db142b47a306f567ad7
Parents: 8e65339
Author: Kurt Young <yk...@gmail.com>
Authored: Thu Sep 29 08:56:27 2016 +0800
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:29 2016 +0100
----------------------------------------------------------------------
.../runtime/jobmaster/JobManagerRunner.java | 8 +-
.../flink/runtime/jobmaster/JobMaster.java | 222 +++++++++++++++++--
.../runtime/jobmaster/JobMasterGateway.java | 17 +-
.../jobmaster/JobMasterRegistrationSuccess.java | 18 +-
.../JobMasterToResourceManagerConnection.java | 117 ----------
.../resourcemanager/ResourceManager.java | 2 +-
.../jobmaster/JobManagerRunnerMockTest.java | 25 ++-
7 files changed, 239 insertions(+), 170 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/dd8c9b9e/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
index bc2bf9a..6944d85 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
@@ -63,9 +63,6 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
private final JobMaster jobManager;
- /** Leader session id when granted leadership */
- private UUID leaderSessionID;
-
/** flag marking the runner as shut down */
private volatile boolean shutdown;
@@ -93,7 +90,6 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
this.executionContext = rpcService.getExecutor();
this.checkpointRecoveryFactory = haServices.getCheckpointRecoveryFactory();
this.leaderElectionService = haServices.getJobMasterLeaderElectionService(jobGraph.getJobID());
- this.leaderSessionID = null;
this.jobManager = new JobMaster(
jobGraph, configuration, rpcService, haServices,
@@ -232,7 +228,6 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
// The operation may be blocking, but since this runner is idle before it been granted the leadership,
// it's okay that job manager wait for the operation complete
leaderElectionService.confirmLeaderSessionID(leaderSessionID);
- this.leaderSessionID = leaderSessionID;
// Double check the leadership after we confirm that, there is a small chance that multiple
// job managers schedule the same job after if they try to recover at the same time.
@@ -242,7 +237,7 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
log.info("Job {} ({}) already finished by others.", jobGraph.getName(), jobGraph.getJobID());
jobFinishedByOther();
} else {
- jobManager.getSelf().startJob();
+ jobManager.getSelf().startJob(leaderSessionID);
}
}
}
@@ -259,7 +254,6 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
log.info("JobManager for job {} ({}) was revoked leadership at {}.",
jobGraph.getName(), jobGraph.getJobID(), getAddress());
- leaderSessionID = null;
jobManager.getSelf().suspendJob(new Exception("JobManager is no longer the leader."));
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/dd8c9b9e/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index b52a23c..1e01c55 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -19,6 +19,7 @@
package org.apache.flink.runtime.jobmaster;
import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.ConfigConstants;
@@ -34,6 +35,7 @@ import org.apache.flink.runtime.checkpoint.stats.DisabledCheckpointStatsTracker;
import org.apache.flink.runtime.checkpoint.stats.SimpleCheckpointStatsTracker;
import org.apache.flink.runtime.client.JobExecutionException;
import org.apache.flink.runtime.client.JobSubmissionException;
+import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager;
import org.apache.flink.runtime.executiongraph.ExecutionGraph;
import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
@@ -47,18 +49,26 @@ import org.apache.flink.runtime.jobgraph.jsonplan.JsonPlanGenerator;
import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
import org.apache.flink.runtime.jobmanager.OnCompletionActions;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup;
+import org.apache.flink.runtime.registration.RegisteredRpcConnection;
+import org.apache.flink.runtime.registration.RegistrationResponse;
+import org.apache.flink.runtime.registration.RetryingRegistration;
import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
+import org.slf4j.Logger;
+
import scala.concurrent.ExecutionContext$;
import scala.concurrent.duration.FiniteDuration;
import java.util.ArrayList;
import java.util.List;
+import java.util.UUID;
import java.util.concurrent.Executor;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -76,9 +86,6 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
*/
public class JobMaster extends RpcEndpoint<JobMasterGateway> {
- /** Gateway to connected resource manager, null iff not connected */
- private ResourceManagerGateway resourceManager = null;
-
/** Logical representation of the job */
private final JobGraph jobGraph;
@@ -123,6 +130,18 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
private MetricGroup jobMetrics;
+ private volatile UUID leaderSessionID;
+
+ // --------- resource manager --------
+
+ /** Leader retriever service used to locate ResourceManager's address */
+ private LeaderRetrievalService resourceManagerLeaderRetriever;
+
+ /** Connection with ResourceManager, null if not located address yet or we close it initiative */
+ private volatile ResourceManagerConnection resourceManagerConnection;
+
+ // ------------------------------------------------------------------------
+
public JobMaster(
JobGraph jobGraph,
Configuration configuration,
@@ -151,10 +170,6 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
this.jobCompletionActions = checkNotNull(jobCompletionActions);
}
- public ResourceManagerGateway getResourceManager() {
- return resourceManager;
- }
-
//----------------------------------------------------------------------------------------------
// Lifecycle management
//----------------------------------------------------------------------------------------------
@@ -196,7 +211,8 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
.getRestartStrategy();
if (restartStrategyConfiguration != null) {
restartStrategy = RestartStrategyFactory.createRestartStrategy(restartStrategyConfiguration);
- } else {
+ }
+ else {
restartStrategy = restartStrategyFactory.createRestartStrategy();
}
@@ -216,6 +232,13 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
throw new JobSubmissionException(jobGraph.getJobID(), "Could not get the checkpoint recovery factory.", e);
}
+ try {
+ resourceManagerLeaderRetriever = highAvailabilityServices.getResourceManagerLeaderRetriever();
+ } catch (Exception e) {
+ log.error("Could not get the resource manager leader retriever.", e);
+ throw new JobSubmissionException(jobGraph.getJobID(),
+ "Could not get the resource manager leader retriever.", e);
+ }
} catch (Throwable t) {
log.error("Failed to initializing job {} ({})", jobGraph.getName(), jobGraph.getJobID(), t);
@@ -223,7 +246,8 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
if (t instanceof JobSubmissionException) {
throw (JobSubmissionException) t;
- } else {
+ }
+ else {
throw new JobSubmissionException(jobGraph.getJobID(), "Failed to initialize job " +
jobGraph.getName() + " (" + jobGraph.getJobID() + ")", t);
}
@@ -240,8 +264,12 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
super.shutDown();
suspendJob(new Exception("JobManager is shutting down."));
+
+ disposeCommunicationWithResourceManager();
}
+
+
//----------------------------------------------------------------------------------------------
// RPC methods
//----------------------------------------------------------------------------------------------
@@ -251,8 +279,10 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
* being recovered. After this, we will begin to schedule the job.
*/
@RpcMethod
- public void startJob() {
- log.info("Starting job {} ({}).", jobGraph.getName(), jobGraph.getJobID());
+ public void startJob(final UUID leaderSessionID) {
+ log.info("Starting job {} ({}) with leaderId {}.", jobGraph.getName(), jobGraph.getJobID(), leaderSessionID);
+
+ this.leaderSessionID = leaderSessionID;
if (executionGraph != null) {
executionGraph = new ExecutionGraph(
@@ -267,7 +297,8 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
jobGraph.getClasspaths(),
userCodeLoader,
jobMetrics);
- } else {
+ }
+ else {
// TODO: update last active time in JobInfo
}
@@ -343,7 +374,8 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
final CheckpointStatsTracker checkpointStatsTracker;
if (isStatsDisabled) {
checkpointStatsTracker = new DisabledCheckpointStatsTracker();
- } else {
+ }
+ else {
int historySize = configuration.getInteger(
ConfigConstants.JOB_MANAGER_WEB_CHECKPOINTS_HISTORY_SIZE,
ConfigConstants.DEFAULT_JOB_MANAGER_WEB_CHECKPOINTS_HISTORY_SIZE);
@@ -397,6 +429,8 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
}
*/
+ // job is good to go, try to locate resource manager's address
+ resourceManagerLeaderRetriever.start(new ResourceManagerLeaderListener());
} catch (Throwable t) {
log.error("Failed to start job {} ({})", jobGraph.getName(), jobGraph.getJobID(), t);
@@ -406,7 +440,8 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
final Throwable rt;
if (t instanceof JobExecutionException) {
rt = (JobExecutionException) t;
- } else {
+ }
+ else {
rt = new JobExecutionException(jobGraph.getJobID(),
"Failed to start job " + jobGraph.getJobID() + " (" + jobGraph.getName() + ")", t);
}
@@ -439,10 +474,14 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
*/
@RpcMethod
public void suspendJob(final Throwable cause) {
+ leaderSessionID = null;
+
if (executionGraph != null) {
executionGraph.suspend(cause);
executionGraph = null;
}
+
+ disposeCommunicationWithResourceManager();
}
/**
@@ -457,14 +496,90 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
return Acknowledge.get();
}
- /**
- * Triggers the registration of the job master at the resource manager.
- *
- * @param address Address of the resource manager
- */
- @RpcMethod
- public void registerAtResourceManager(final String address) {
- //TODO:: register at the RM
+ //----------------------------------------------------------------------------------------------\u2028
+ // Internal methods\u2028
+ // ----------------------------------------------------------------------------------------------\u2028\u2028
+
+ private void handleFatalError(final Throwable cause) {
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ log.error("Fatal error occurred on JobManager, cause: {}", cause.getMessage(), cause);
+ shutDown();
+ jobCompletionActions.onFatalError(cause);
+ }
+ });
+ }
+
+ private void notifyOfNewResourceManagerLeader(
+ final String resourceManagerAddress, final UUID resourceManagerLeaderId)
+ {
+ // IMPORTANT: executed by main thread to avoid concurrence
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ if (resourceManagerConnection != null) {
+ if (resourceManagerAddress != null) {
+ if (resourceManagerAddress.equals(resourceManagerConnection.getTargetAddress())
+ && resourceManagerLeaderId.equals(resourceManagerConnection.getTargetLeaderId()))
+ {
+ // both address and leader id are not changed, we can keep the old connection
+ return;
+ }
+ log.info("ResourceManager leader changed from {} to {}. Registering at new leader.",
+ resourceManagerConnection.getTargetAddress(), resourceManagerAddress);
+ }
+ else {
+ log.info("Current ResourceManager {} lost leader status. Waiting for new ResourceManager leader.",
+ resourceManagerConnection.getTargetAddress());
+ }
+ }
+
+ closeResourceManagerConnection();
+
+ if (resourceManagerAddress != null) {
+ log.info("Attempting to register at ResourceManager {}", resourceManagerAddress);
+ resourceManagerConnection = new ResourceManagerConnection(
+ log, jobGraph.getJobID(), leaderSessionID,
+ resourceManagerAddress, resourceManagerLeaderId, executionContext);
+ resourceManagerConnection.start();
+ }
+ }
+ });
+ }
+
+ private void onResourceManagerRegistrationSuccess(final JobMasterRegistrationSuccess success) {
+ getRpcService().execute(new Runnable() {
+ @Override
+ public void run() {
+ // TODO - add tests for comment in https://github.com/apache/flink/pull/2565
+ // verify the response with current connection
+ if (resourceManagerConnection != null
+ && resourceManagerConnection.getTargetLeaderId().equals(success.getResourceManagerLeaderId())) {
+ log.info("JobManager successfully registered at ResourceManager, leader id: {}.",
+ success.getResourceManagerLeaderId());
+ }
+ }
+ });
+ }
+
+ private void disposeCommunicationWithResourceManager() {
+ // 1. stop the leader retriever so we will not receiving updates anymore
+ try {
+ resourceManagerLeaderRetriever.stop();
+ } catch (Exception e) {
+ log.warn("Failed to stop resource manager leader retriever.");
+ }
+
+ // 2. close current connection with ResourceManager if exists
+ closeResourceManagerConnection();
+ }
+
+ private void closeResourceManagerConnection() {
+ if (resourceManagerConnection != null) {
+ resourceManagerConnection.close();
+ resourceManagerConnection = null;
+ }
}
//----------------------------------------------------------------------------------------------
@@ -494,4 +609,67 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
}
return ret;
}
+
+ //----------------------------------------------------------------------------------------------
+ // Utility classes
+ //----------------------------------------------------------------------------------------------
+
+ private class ResourceManagerLeaderListener implements LeaderRetrievalListener {
+ @Override
+ public void notifyLeaderAddress(final String leaderAddress, final UUID leaderSessionID) {
+ notifyOfNewResourceManagerLeader(leaderAddress, leaderSessionID);
+ }
+
+ @Override
+ public void handleError(final Exception exception) {
+ handleFatalError(exception);
+ }
+ }
+
+ private class ResourceManagerConnection
+ extends RegisteredRpcConnection<ResourceManagerGateway, JobMasterRegistrationSuccess>
+ {
+ private final JobID jobID;
+
+ private final UUID jobManagerLeaderID;
+
+ ResourceManagerConnection(
+ final Logger log,
+ final JobID jobID,
+ final UUID jobManagerLeaderID,
+ final String resourceManagerAddress,
+ final UUID resourceManagerLeaderID,
+ final Executor executor)
+ {
+ super(log, resourceManagerAddress, resourceManagerLeaderID, executor);
+ this.jobID = checkNotNull(jobID);
+ this.jobManagerLeaderID = checkNotNull(jobManagerLeaderID);
+ }
+
+ @Override
+ protected RetryingRegistration<ResourceManagerGateway, JobMasterRegistrationSuccess> generateRegistration() {
+ return new RetryingRegistration<ResourceManagerGateway, JobMasterRegistrationSuccess>(
+ log, getRpcService(), "ResourceManager", ResourceManagerGateway.class,
+ getTargetAddress(), getTargetLeaderId())
+ {
+ @Override
+ protected Future<RegistrationResponse> invokeRegistration(ResourceManagerGateway gateway, UUID leaderId,
+ long timeoutMillis) throws Exception
+ {
+ Time timeout = Time.milliseconds(timeoutMillis);
+ return gateway.registerJobMaster(leaderId, jobManagerLeaderID, getAddress(), jobID, timeout);
+ }
+ };
+ }
+
+ @Override
+ protected void onRegistrationSuccess(final JobMasterRegistrationSuccess success) {
+ onResourceManagerRegistrationSuccess(success);
+ }
+
+ @Override
+ protected void onRegistrationFailure(final Throwable failure) {
+ handleFatalError(failure);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/dd8c9b9e/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
index b281ea8..6587ccb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
@@ -23,19 +23,21 @@ import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.rpc.RpcGateway;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
+import java.util.UUID;
+
/**
* {@link JobMaster} rpc gateway interface
*/
public interface JobMasterGateway extends RpcGateway {
/**
- * Making this job begins to run.
+ * Starting the job under the given leader session ID.
*/
- void startJob();
+ void startJob(final UUID leaderSessionID);
/**
- * Suspending job, all the running tasks will be cancelled, and runtime status will be cleared. Should re-submit
- * the job before restarting it.
+ * Suspending job, all the running tasks will be cancelled, and runtime status will be cleared.
+ * Should re-submit the job before restarting it.
*
* @param cause The reason of why this job been suspended.
*/
@@ -48,11 +50,4 @@ public interface JobMasterGateway extends RpcGateway {
* @return Future acknowledge of the task execution state update
*/
Future<Acknowledge> updateTaskExecutionState(TaskExecutionState taskExecutionState);
-
- /**
- * Triggers the registration of the job master at the resource manager.
- *
- * @param address Address of the resource manager
- */
- void registerAtResourceManager(final String address);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/dd8c9b9e/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterRegistrationSuccess.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterRegistrationSuccess.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterRegistrationSuccess.java
index 031c38e..4058452 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterRegistrationSuccess.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterRegistrationSuccess.java
@@ -20,6 +20,10 @@ package org.apache.flink.runtime.jobmaster;
import org.apache.flink.runtime.registration.RegistrationResponse;
+import java.util.UUID;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* Base class for responses from the ResourceManager to a registration attempt by a JobMaster.
*/
@@ -29,8 +33,11 @@ public class JobMasterRegistrationSuccess extends RegistrationResponse.Success {
private final long heartbeatInterval;
- public JobMasterRegistrationSuccess(long heartbeatInterval) {
+ private final UUID resourceManagerLeaderId;
+
+ public JobMasterRegistrationSuccess(final long heartbeatInterval, final UUID resourceManagerLeaderId) {
this.heartbeatInterval = heartbeatInterval;
+ this.resourceManagerLeaderId = checkNotNull(resourceManagerLeaderId);
}
/**
@@ -42,8 +49,15 @@ public class JobMasterRegistrationSuccess extends RegistrationResponse.Success {
return heartbeatInterval;
}
+ public UUID getResourceManagerLeaderId() {
+ return resourceManagerLeaderId;
+ }
+
@Override
public String toString() {
- return "JobMasterRegistrationSuccess(" + heartbeatInterval + ')';
+ return "JobMasterRegistrationSuccess{" +
+ "heartbeatInterval=" + heartbeatInterval +
+ ", resourceManagerLeaderId=" + resourceManagerLeaderId +
+ '}';
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/dd8c9b9e/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterToResourceManagerConnection.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterToResourceManagerConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterToResourceManagerConnection.java
deleted file mode 100644
index 71fce8c..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterToResourceManagerConnection.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.jobmaster;
-
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.time.Time;
-import org.apache.flink.runtime.registration.RegisteredRpcConnection;
-import org.apache.flink.runtime.registration.RegistrationResponse;
-import org.apache.flink.runtime.registration.RetryingRegistration;
-import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
-import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.runtime.concurrent.Future;
-
-import org.slf4j.Logger;
-
-import java.util.UUID;
-import java.util.concurrent.Executor;
-
-import static org.apache.flink.util.Preconditions.checkNotNull;
-
-/**
- * The connection between a JobMaster and the ResourceManager.
- */
-public class JobMasterToResourceManagerConnection
- extends RegisteredRpcConnection<ResourceManagerGateway, JobMasterRegistrationSuccess> {
-
- /** the JobMaster whose connection to the ResourceManager this represents */
- private final JobMaster jobMaster;
-
- private final JobID jobID;
-
- private final UUID jobMasterLeaderId;
-
- public JobMasterToResourceManagerConnection(
- Logger log,
- JobID jobID,
- JobMaster jobMaster,
- UUID jobMasterLeaderId,
- String resourceManagerAddress,
- UUID resourceManagerLeaderId,
- Executor executor) {
-
- super(log, resourceManagerAddress, resourceManagerLeaderId, executor);
- this.jobMaster = checkNotNull(jobMaster);
- this.jobID = checkNotNull(jobID);
- this.jobMasterLeaderId = checkNotNull(jobMasterLeaderId);
- }
-
- @Override
- protected RetryingRegistration<ResourceManagerGateway, JobMasterRegistrationSuccess> generateRegistration() {
- return new JobMasterToResourceManagerConnection.ResourceManagerRegistration(
- log, jobMaster.getRpcService(),
- getTargetAddress(), getTargetLeaderId(),
- jobMaster.getAddress(),jobID, jobMasterLeaderId);
- }
-
- @Override
- protected void onRegistrationSuccess(JobMasterRegistrationSuccess success) {
- }
-
- @Override
- protected void onRegistrationFailure(Throwable failure) {
- }
-
- // ------------------------------------------------------------------------
- // Utilities
- // ------------------------------------------------------------------------
-
- private static class ResourceManagerRegistration
- extends RetryingRegistration<ResourceManagerGateway, JobMasterRegistrationSuccess> {
-
- private final String jobMasterAddress;
-
- private final JobID jobID;
-
- private final UUID jobMasterLeaderId;
-
- ResourceManagerRegistration(
- Logger log,
- RpcService rpcService,
- String targetAddress,
- UUID leaderId,
- String jobMasterAddress,
- JobID jobID,
- UUID jobMasterLeaderId) {
-
- super(log, rpcService, "ResourceManager", ResourceManagerGateway.class, targetAddress, leaderId);
- this.jobMasterAddress = checkNotNull(jobMasterAddress);
- this.jobID = checkNotNull(jobID);
- this.jobMasterLeaderId = checkNotNull(jobMasterLeaderId);
- }
-
- @Override
- protected Future<RegistrationResponse> invokeRegistration(
- ResourceManagerGateway gateway, UUID leaderId, long timeoutMillis) throws Exception {
-
- Time timeout = Time.milliseconds(timeoutMillis);
- return gateway.registerJobMaster(leaderId, jobMasterLeaderId,jobMasterAddress, jobID, timeout);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/dd8c9b9e/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index 190a4de..f695de4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -215,7 +215,7 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
if (existingGateway != null) {
log.info("Replacing gateway for registered JobID {}.", jobID);
}
- return new JobMasterRegistrationSuccess(5000);
+ return new JobMasterRegistrationSuccess(5000, resourceManagerLeaderId);
}
}
}, getMainThreadExecutor());
http://git-wip-us.apache.org/repos/asf/flink/blob/dd8c9b9e/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
index dc3b5fd..bfe5f55 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
@@ -142,8 +142,9 @@ public class JobManagerRunnerMockTest {
public void testJobFinished() throws Exception {
runner.start();
- runner.grantLeadership(UUID.randomUUID());
- verify(jobManagerGateway).startJob();
+ UUID leaderSessionID = UUID.randomUUID();
+ runner.grantLeadership(leaderSessionID);
+ verify(jobManagerGateway).startJob(leaderSessionID);
assertTrue(!jobCompletion.isJobFinished());
// runner been told by JobManager that job is finished
@@ -160,8 +161,9 @@ public class JobManagerRunnerMockTest {
public void testJobFailed() throws Exception {
runner.start();
- runner.grantLeadership(UUID.randomUUID());
- verify(jobManagerGateway).startJob();
+ UUID leaderSessionID = UUID.randomUUID();
+ runner.grantLeadership(leaderSessionID);
+ verify(jobManagerGateway).startJob(leaderSessionID);
assertTrue(!jobCompletion.isJobFinished());
// runner been told by JobManager that job is failed
@@ -177,8 +179,9 @@ public class JobManagerRunnerMockTest {
public void testLeadershipRevoked() throws Exception {
runner.start();
- runner.grantLeadership(UUID.randomUUID());
- verify(jobManagerGateway).startJob();
+ UUID leaderSessionID = UUID.randomUUID();
+ runner.grantLeadership(leaderSessionID);
+ verify(jobManagerGateway).startJob(leaderSessionID);
assertTrue(!jobCompletion.isJobFinished());
runner.revokeLeadership();
@@ -190,16 +193,18 @@ public class JobManagerRunnerMockTest {
public void testRegainLeadership() throws Exception {
runner.start();
- runner.grantLeadership(UUID.randomUUID());
- verify(jobManagerGateway).startJob();
+ UUID leaderSessionID = UUID.randomUUID();
+ runner.grantLeadership(leaderSessionID);
+ verify(jobManagerGateway).startJob(leaderSessionID);
assertTrue(!jobCompletion.isJobFinished());
runner.revokeLeadership();
verify(jobManagerGateway).suspendJob(any(Throwable.class));
assertFalse(runner.isShutdown());
- runner.grantLeadership(UUID.randomUUID());
- verify(jobManagerGateway, times(2)).startJob();
+ UUID leaderSessionID2 = UUID.randomUUID();
+ runner.grantLeadership(leaderSessionID2);
+ verify(jobManagerGateway, times(2)).startJob(leaderSessionID2);
}
private static class TestingOnCompletionActions implements OnCompletionActions {
[31/50] [abbrv] flink git commit: [FLINK-4489] [tm] Add TaskSlotTable
to manage slot allocations for multiple job managers
Posted by tr...@apache.org.
[FLINK-4489] [tm] Add TaskSlotTable to manage slot allocations for multiple job managers
Add TimerService for slot timeouts
Add task and task slot access methods
Add comments to newly introduced classes
This closes #2638.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a122bfd8
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a122bfd8
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a122bfd8
Branch: refs/heads/flip-6
Commit: a122bfd8215ce48b44b4eb9f140be0a272f77d2f
Parents: 1801de9
Author: Till Rohrmann <tr...@apache.org>
Authored: Wed Oct 5 11:58:26 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:31 2016 +0100
----------------------------------------------------------------------
.../runtime/clusterframework/types/SlotID.java | 4 +-
.../resourcemanager/ResourceManager.java | 13 +-
.../resourcemanager/ResourceManagerGateway.java | 13 +-
.../runtime/taskexecutor/TaskExecutor.java | 194 +++---
.../runtime/taskexecutor/TaskManagerRunner.java | 1 +
.../taskexecutor/TaskManagerServices.java | 29 +-
.../TaskManagerServicesConfiguration.java | 2 +
.../flink/runtime/taskexecutor/TaskSlot.java | 73 --
.../runtime/taskexecutor/TaskSlotMapping.java | 44 --
.../runtime/taskexecutor/slot/SlotActions.java | 45 ++
.../slot/SlotNotActiveException.java | 34 +
.../slot/SlotNotFoundException.java | 37 +
.../runtime/taskexecutor/slot/TaskSlot.java | 289 ++++++++
.../taskexecutor/slot/TaskSlotState.java | 29 +
.../taskexecutor/slot/TaskSlotTable.java | 682 +++++++++++++++++++
.../taskexecutor/slot/TimeoutListener.java | 37 +
.../runtime/taskexecutor/slot/TimerService.java | 160 +++++
.../apache/flink/runtime/taskmanager/Task.java | 8 +-
.../PartialConsumePipelinedResultTest.java | 1 -
.../runtime/taskexecutor/TaskExecutorTest.java | 4 +
20 files changed, 1457 insertions(+), 242 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
index 237597b..d6409b6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
@@ -20,6 +20,7 @@ package org.apache.flink.runtime.clusterframework.types;
import java.io.Serializable;
+import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
@@ -34,8 +35,9 @@ public class SlotID implements ResourceIDRetrievable, Serializable {
/** The numeric id for single slot */
private final int slotNumber;
-
+
public SlotID(ResourceID resourceId, int slotNumber) {
+ checkArgument(0 <= slotNumber, "Slot number must be positive.");
this.resourceId = checkNotNull(resourceId, "ResourceID must not be null");
this.slotNumber = slotNumber;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index 8fbb34b..3122804 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -40,7 +40,6 @@ import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestRejected;
import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestReply;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.SlotAvailableReply;
import org.apache.flink.runtime.resourcemanager.registration.JobMasterRegistration;
import org.apache.flink.runtime.resourcemanager.registration.WorkerRegistration;
import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
@@ -337,34 +336,32 @@ public abstract class ResourceManager<WorkerType extends Serializable>
/**
* Notification from a TaskExecutor that a slot has become available
* @param resourceManagerLeaderId TaskExecutor's resource manager leader id
- * @param resourceID TaskExecutor's resource id
* @param instanceID TaskExecutor's instance id
* @param slotID The slot id of the available slot
* @return SlotAvailableReply
*/
@RpcMethod
- public SlotAvailableReply notifySlotAvailable(
+ public void notifySlotAvailable(
final UUID resourceManagerLeaderId,
- final ResourceID resourceID,
final InstanceID instanceID,
final SlotID slotID) {
if (resourceManagerLeaderId.equals(leaderSessionID)) {
- WorkerRegistration<WorkerType> registration = taskExecutors.get(resourceID);
+ final ResourceID resourceId = slotID.getResourceID();
+ WorkerRegistration<WorkerType> registration = taskExecutors.get(resourceId);
+
if (registration != null) {
InstanceID registrationInstanceID = registration.getInstanceID();
if (registrationInstanceID.equals(instanceID)) {
runAsync(new Runnable() {
@Override
public void run() {
- slotManager.notifySlotAvailable(resourceID, slotID);
+ slotManager.notifySlotAvailable(resourceId, slotID);
}
});
- return new SlotAvailableReply(leaderSessionID, slotID);
}
}
}
- return null;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
index 07e9e43..968eeb8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
@@ -26,7 +26,6 @@ import org.apache.flink.runtime.clusterframework.types.SlotID;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.instance.InstanceID;
import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestReply;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.SlotAvailableReply;
import org.apache.flink.runtime.rpc.RpcGateway;
import org.apache.flink.runtime.rpc.RpcTimeout;
import org.apache.flink.runtime.jobmaster.JobMaster;
@@ -94,17 +93,13 @@ public interface ResourceManagerGateway extends RpcGateway {
* Sent by the TaskExecutor to notify the ResourceManager that a slot has become available.
*
* @param resourceManagerLeaderId The ResourceManager leader id
- * @param resourceID The ResourceID of the TaskExecutor
- * @param instanceID The InstanceID of the TaskExecutor
+ * @param instanceId TaskExecutor's instance id
* @param slotID The SlotID of the freed slot
- * @return The confirmation by the ResourceManager
*/
- Future<SlotAvailableReply> notifySlotAvailable(
+ void notifySlotAvailable(
UUID resourceManagerLeaderId,
- ResourceID resourceID,
- InstanceID instanceID,
- SlotID slotID,
- @RpcTimeout Time timeout);
+ InstanceID instanceId,
+ SlotID slotID);
/**
* Registers an infoMessage listener
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index 9f9234f..e642315 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -47,6 +47,7 @@ import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
import org.apache.flink.runtime.memory.MemoryManager;
import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRegistered;
@@ -64,6 +65,10 @@ import org.apache.flink.runtime.taskexecutor.rpc.RpcCheckpointResponder;
import org.apache.flink.runtime.taskexecutor.rpc.RpcInputSplitProvider;
import org.apache.flink.runtime.taskexecutor.rpc.RpcPartitionStateChecker;
import org.apache.flink.runtime.taskexecutor.rpc.RpcResultPartitionConsumableNotifier;
+import org.apache.flink.runtime.taskexecutor.slot.SlotActions;
+import org.apache.flink.runtime.taskexecutor.slot.SlotNotActiveException;
+import org.apache.flink.runtime.taskexecutor.slot.SlotNotFoundException;
+import org.apache.flink.runtime.taskexecutor.slot.TaskSlotTable;
import org.apache.flink.runtime.taskmanager.CheckpointResponder;
import org.apache.flink.runtime.taskmanager.Task;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
@@ -76,7 +81,6 @@ import java.net.InetSocketAddress;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
@@ -132,13 +136,9 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
private Map<ResourceID, JobManagerConnection> jobManagerConnections;
- // --------- Slot allocation table --------
+ // --------- task slot allocation table -----------
- private Map<AllocationID, TaskSlot> taskSlots;
-
- // --------- Slot allocation table --------
-
- private Map<ExecutionAttemptID, TaskSlotMapping> taskSlotMappings;
+ private final TaskSlotTable taskSlotTable;
// ------------------------------------------------------------------------
@@ -154,6 +154,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
TaskManagerMetricGroup taskManagerMetricGroup,
BroadcastVariableManager broadcastVariableManager,
FileCache fileCache,
+ TaskSlotTable taskSlotTable,
FatalErrorHandler fatalErrorHandler) {
super(rpcService);
@@ -167,6 +168,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
this.networkEnvironment = checkNotNull(networkEnvironment);
this.haServices = checkNotNull(haServices);
this.metricRegistry = checkNotNull(metricRegistry);
+ this.taskSlotTable = checkNotNull(taskSlotTable);
this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
this.taskManagerMetricGroup = checkNotNull(taskManagerMetricGroup);
this.broadcastVariableManager = checkNotNull(broadcastVariableManager);
@@ -175,8 +177,6 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
this.jobManagerConnections = new HashMap<>(4);
this.unconfirmedFreeSlots = new HashSet<>();
- this.taskSlots = new HashMap<>(taskManagerConfiguration.getNumberSlots());
- this.taskSlotMappings = new HashMap<>(taskManagerConfiguration.getNumberSlots() * 2);
}
// ------------------------------------------------------------------------
@@ -193,6 +193,9 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
} catch (Exception e) {
onFatalErrorAsync(e);
}
+
+ // tell the task slot table who's responsible for the task slot actions
+ taskSlotTable.start(new SlotActionsImpl(), taskManagerConfiguration.getTimeout());
}
/**
@@ -202,6 +205,8 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
public void shutDown() {
log.info("Stopping TaskManager {}.", getAddress());
+ taskSlotTable.stop();
+
if (resourceManagerConnection.isConnected()) {
try {
resourceManagerConnection.close();
@@ -264,10 +269,9 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
throw new TaskSubmissionException(message);
}
- TaskSlot taskSlot = taskSlots.get(tdd.getAllocationID());
-
- if (taskSlot == null) {
- final String message = "No task slot allocated for allocation ID " + tdd.getAllocationID() + '.';
+ if (!taskSlotTable.existActiveSlot(tdd.getJobID(), tdd.getAllocationID())) {
+ final String message = "No task slot allocated for job ID " + tdd.getJobID() +
+ " and allocation ID " + tdd.getAllocationID() + '.';
log.debug(message);
throw new TaskSubmissionException(message);
}
@@ -307,10 +311,15 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
log.info("Received task {}.", task.getTaskInfo().getTaskNameWithSubtasks());
- if(taskSlot.add(task)) {
- TaskSlotMapping taskSlotMapping = new TaskSlotMapping(task, taskSlot);
+ boolean taskAdded;
- taskSlotMappings.put(task.getExecutionId(), taskSlotMapping);
+ try {
+ taskAdded = taskSlotTable.addTask(task);
+ } catch (SlotNotFoundException | SlotNotActiveException e) {
+ throw new TaskSubmissionException("Could not submit task.", e);
+ }
+
+ if (taskAdded) {
task.startTaskThread();
return Acknowledge.get();
@@ -325,7 +334,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
@RpcMethod
public Acknowledge cancelTask(ExecutionAttemptID executionAttemptID) throws TaskException {
- final Task task = getTask(executionAttemptID);
+ final Task task = taskSlotTable.getTask(executionAttemptID);
if (task != null) {
try {
@@ -344,7 +353,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
@RpcMethod
public Acknowledge stopTask(ExecutionAttemptID executionAttemptID) throws TaskException {
- final Task task = getTask(executionAttemptID);
+ final Task task = taskSlotTable.getTask(executionAttemptID);
if (task != null) {
try {
@@ -367,7 +376,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
@RpcMethod
public Acknowledge updatePartitions(final ExecutionAttemptID executionAttemptID, Collection<PartitionInfo> partitionInfos) throws PartitionException {
- final Task task = getTask(executionAttemptID);
+ final Task task = taskSlotTable.getTask(executionAttemptID);
if (task != null) {
for (final PartitionInfo partitionInfo: partitionInfos) {
@@ -430,7 +439,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
public Acknowledge triggerCheckpoint(ExecutionAttemptID executionAttemptID, long checkpointId, long checkpointTimestamp) throws CheckpointException {
log.debug("Trigger checkpoint {}@{} for {}.", checkpointId, checkpointTimestamp, executionAttemptID);
- final Task task = getTask(executionAttemptID);
+ final Task task = taskSlotTable.getTask(executionAttemptID);
if (task != null) {
task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp);
@@ -448,7 +457,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
public Acknowledge confirmCheckpoint(ExecutionAttemptID executionAttemptID, long checkpointId, long checkpointTimestamp) throws CheckpointException {
log.debug("Confirm checkpoint {}@{} for {}.", checkpointId, checkpointTimestamp, executionAttemptID);
- final Task task = getTask(executionAttemptID);
+ final Task task = taskSlotTable.getTask(executionAttemptID);
if (task != null) {
task.notifyCheckpointComplete(checkpointId);
@@ -494,68 +503,8 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
return jobManagerConnections.get(jobManagerID);
}
- private Task getTask(ExecutionAttemptID executionAttemptID) {
- TaskSlotMapping taskSlotMapping = taskSlotMappings.get(executionAttemptID);
-
- if (taskSlotMapping != null) {
- return taskSlotMapping.getTask();
- } else {
- return null;
- }
- }
-
- private Task removeTask(ExecutionAttemptID executionAttemptID) {
- TaskSlotMapping taskSlotMapping = taskSlotMappings.remove(executionAttemptID);
-
- if (taskSlotMapping != null) {
- final Task task = taskSlotMapping.getTask();
- final TaskSlot taskSlot = taskSlotMapping.getTaskSlot();
-
- taskSlot.remove(task);
-
- return task;
- } else {
- return null;
- }
- }
-
- private Iterable<Task> getAllTasks() {
- final Iterator<TaskSlotMapping> taskEntryIterator = taskSlotMappings.values().iterator();
- final Iterator<Task> iterator = new Iterator<Task>() {
- @Override
- public boolean hasNext() {
- return taskEntryIterator.hasNext();
- }
-
- @Override
- public Task next() {
- return taskEntryIterator.next().getTask();
- }
-
- @Override
- public void remove() {
- taskEntryIterator.remove();
- }
- };
-
- return new Iterable<Task>() {
- @Override
- public Iterator<Task> iterator() {
- return iterator;
- }
- };
- }
-
- private void clearTasks() {
- taskSlotMappings.clear();
-
- for (TaskSlot taskSlot: taskSlots.values()) {
- taskSlot.clear();
- }
- }
-
private void failTask(final ExecutionAttemptID executionAttemptID, final Throwable cause) {
- final Task task = getTask(executionAttemptID);
+ final Task task = taskSlotTable.getTask(executionAttemptID);
if (task != null) {
try {
@@ -568,18 +517,6 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
- private void cancelAndClearAllTasks(Throwable cause) {
- log.info("Cancellaing all computations and discarding all cached data.");
-
- Iterable<Task> tasks = getAllTasks();
-
- for (Task task: tasks) {
- task.failExternally(cause);
- }
-
- clearTasks();
- }
-
private void updateTaskExecutionState(
final UUID jobMasterLeaderId,
final JobMasterGateway jobMasterGateway,
@@ -602,11 +539,10 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
private void unregisterTaskAndNotifyFinalState(
final UUID jobMasterLeaderId,
- final JobMasterGateway jobMasterGateway,
- final ExecutionAttemptID executionAttemptID)
- {
- Task task = removeTask(executionAttemptID);
+ final JobMasterGateway jobMasterGateway,
+ final ExecutionAttemptID executionAttemptID) {
+ Task task = taskSlotTable.removeTask(executionAttemptID);
if (task != null) {
if (!task.getExecutionState().isTerminal()) {
try {
@@ -718,6 +654,41 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
+ private void freeSlot(AllocationID allocationId) {
+ Preconditions.checkNotNull(allocationId);
+
+ try {
+ int freedSlotIndex = taskSlotTable.freeSlot(allocationId);
+
+ if (freedSlotIndex != -1 && isConnectedToResourceManager()) {
+ // the slot was freed. Tell the RM about it
+ ResourceManagerGateway resourceManagerGateway = resourceManagerConnection.getTargetGateway();
+
+ resourceManagerGateway.notifySlotAvailable(
+ resourceManagerConnection.getTargetLeaderId(),
+ resourceManagerConnection.getRegistrationId(),
+ new SlotID(getResourceID(), freedSlotIndex));
+ }
+ } catch (SlotNotFoundException e) {
+ log.debug("Could not free slot for allocation id {}.", allocationId, e);
+ }
+ }
+
+ private void timeoutSlot(AllocationID allocationId, UUID ticket) {
+ Preconditions.checkNotNull(allocationId);
+ Preconditions.checkNotNull(ticket);
+
+ if (taskSlotTable.isValidTimeout(allocationId, ticket)) {
+ freeSlot(allocationId);
+ } else {
+ log.debug("Received an invalid timeout for allocation id {} with ticket {}.", allocationId, ticket);
+ }
+ }
+
+ private boolean isConnectedToResourceManager() {
+ return (resourceManagerConnection != null && resourceManagerConnection.isConnected());
+ }
+
// ------------------------------------------------------------------------
// Properties
// ------------------------------------------------------------------------
@@ -778,7 +749,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
/**
* The listener for leader changes of the resource manager
*/
- private class ResourceManagerLeaderListener implements LeaderRetrievalListener {
+ private final class ResourceManagerLeaderListener implements LeaderRetrievalListener {
@Override
public void notifyLeaderAddress(final String leaderAddress, final UUID leaderSessionID) {
@@ -796,7 +767,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
- private class TaskManagerActionsImpl implements TaskManagerActions {
+ private final class TaskManagerActionsImpl implements TaskManagerActions {
private final UUID jobMasterLeaderId;
private final JobMasterGateway jobMasterGateway;
@@ -837,4 +808,27 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
+ private class SlotActionsImpl implements SlotActions {
+
+ @Override
+ public void freeSlot(final AllocationID allocationId) {
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ TaskExecutor.this.freeSlot(allocationId);
+ }
+ });
+ }
+
+ @Override
+ public void timeoutSlot(final AllocationID allocationId, final UUID ticket) {
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ TaskExecutor.this.timeoutSlot(allocationId, ticket);
+ }
+ });
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
index bb66655..ca1d2ce 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
@@ -98,6 +98,7 @@ public class TaskManagerRunner implements FatalErrorHandler {
taskManagerServices.getTaskManagerMetricGroup(),
taskManagerServices.getBroadcastVariableManager(),
taskManagerServices.getFileCache(),
+ taskManagerServices.getTaskSlotTable(),
this);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
index e264a1c..c1728b4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
@@ -20,7 +20,9 @@ package org.apache.flink.runtime.taskexecutor;
import org.apache.flink.core.memory.MemoryType;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.filecache.FileCache;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync;
@@ -38,6 +40,8 @@ import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
import org.apache.flink.runtime.query.KvStateRegistry;
import org.apache.flink.runtime.query.netty.DisabledKvStateRequestStats;
import org.apache.flink.runtime.query.netty.KvStateServer;
+import org.apache.flink.runtime.taskexecutor.slot.TaskSlotTable;
+import org.apache.flink.runtime.taskexecutor.slot.TimerService;
import org.apache.flink.runtime.taskexecutor.utils.TaskExecutorMetricsInitializer;
import org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
@@ -48,6 +52,9 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
/**
* Container for {@link TaskExecutor} services such as the {@link MemoryManager}, {@link IOManager},
@@ -65,6 +72,7 @@ public class TaskManagerServices {
private final TaskManagerMetricGroup taskManagerMetricGroup;
private final BroadcastVariableManager broadcastVariableManager;
private final FileCache fileCache;
+ private final TaskSlotTable taskSlotTable;
private TaskManagerServices(
TaskManagerLocation taskManagerLocation,
@@ -74,7 +82,8 @@ public class TaskManagerServices {
MetricRegistry metricRegistry,
TaskManagerMetricGroup taskManagerMetricGroup,
BroadcastVariableManager broadcastVariableManager,
- FileCache fileCache) {
+ FileCache fileCache,
+ TaskSlotTable taskSlotTable) {
this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
this.memoryManager = Preconditions.checkNotNull(memoryManager);
@@ -84,6 +93,7 @@ public class TaskManagerServices {
this.taskManagerMetricGroup = Preconditions.checkNotNull(taskManagerMetricGroup);
this.broadcastVariableManager = Preconditions.checkNotNull(broadcastVariableManager);
this.fileCache = Preconditions.checkNotNull(fileCache);
+ this.taskSlotTable = Preconditions.checkNotNull(taskSlotTable);
}
// --------------------------------------------------------------------------------------------
@@ -121,6 +131,10 @@ public class TaskManagerServices {
public FileCache getFileCache() {
return fileCache;
}
+
+ public TaskSlotTable getTaskSlotTable() {
+ return taskSlotTable;
+ }
// --------------------------------------------------------------------------------------------
// Static factory methods for task manager services
@@ -167,6 +181,16 @@ public class TaskManagerServices {
final FileCache fileCache = new FileCache(taskManagerServicesConfiguration.getTmpDirPaths());
+ final List<ResourceProfile> resourceProfiles = new ArrayList<>(taskManagerServicesConfiguration.getNumberOfSlots());
+
+ for (int i = 0; i < taskManagerServicesConfiguration.getNumberOfSlots(); i++) {
+ resourceProfiles.add(new ResourceProfile(1.0, 42L));
+ }
+
+ final TimerService<AllocationID> timerService = new TimerService<>(new ScheduledThreadPoolExecutor(1));
+
+ final TaskSlotTable taskSlotTable = new TaskSlotTable(resourceProfiles, timerService);
+
return new TaskManagerServices(
taskManagerLocation,
memoryManager,
@@ -175,7 +199,8 @@ public class TaskManagerServices {
metricRegistry,
taskManagerMetricGroup,
broadcastVariableManager,
- fileCache);
+ fileCache,
+ taskSlotTable);
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
index 80dfc09..036a890 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java
@@ -173,6 +173,8 @@ public class TaskManagerServicesConfiguration {
final MetricRegistryConfiguration metricRegistryConfiguration = MetricRegistryConfiguration.fromConfiguration(configuration);
+
+
return new TaskManagerServicesConfiguration(
remoteAddress,
tmpDirs,
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlot.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlot.java
deleted file mode 100644
index 4fc1d66..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlot.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.taskexecutor;
-
-import org.apache.flink.runtime.clusterframework.types.AllocationID;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
-import org.apache.flink.runtime.taskmanager.Task;
-import org.apache.flink.util.Preconditions;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Container for multiple {@link Task} belonging to the same slot.
- */
-public class TaskSlot {
- private final AllocationID allocationID;
- private final ResourceID resourceID;
- private final Map<ExecutionAttemptID, Task> tasks;
-
- public TaskSlot(AllocationID allocationID, ResourceID resourceID) {
- this.allocationID = Preconditions.checkNotNull(allocationID);
- this.resourceID = Preconditions.checkNotNull(resourceID);
- tasks = new HashMap<>(4);
- }
-
- public AllocationID getAllocationID() {
- return allocationID;
- }
-
- public ResourceID getResourceID() {
- return resourceID;
- }
-
- public boolean add(Task task) {
- // sanity check
- Preconditions.checkArgument(allocationID.equals(task.getAllocationID()));
-
- Task oldTask = tasks.put(task.getExecutionId(), task);
-
- if (oldTask != null) {
- tasks.put(task.getExecutionId(), oldTask);
- return false;
- } else {
- return true;
- }
- }
-
- public Task remove(Task task) {
- return tasks.remove(task.getExecutionId());
- }
-
- public void clear() {
- tasks.clear();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlotMapping.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlotMapping.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlotMapping.java
deleted file mode 100644
index e67fd52..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskSlotMapping.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.taskexecutor;
-
-import org.apache.flink.runtime.taskmanager.Task;
-import org.apache.flink.util.Preconditions;
-
-/**
- * Mapping between a {@link Task} and its {@link TaskSlot}.
- */
-public class TaskSlotMapping {
-
- private final Task task;
- private final TaskSlot taskSlot;
-
- public TaskSlotMapping(Task task, TaskSlot taskSlot) {
- this.task = Preconditions.checkNotNull(task);
- this.taskSlot = Preconditions.checkNotNull(taskSlot);
- }
-
- public Task getTask() {
- return task;
- }
-
- public TaskSlot getTaskSlot() {
- return taskSlot;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotActions.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotActions.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotActions.java
new file mode 100644
index 0000000..f7ed235
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotActions.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.slot;
+
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+
+import java.util.UUID;
+
+/**
+ * Interface to trigger slot actions from within the {@link TaskSlotTable}.
+ */
+public interface SlotActions {
+
+ /**
+ * Free the task slot with the given allocation id.
+ *
+ * @param allocationId to identify the slot to be freed
+ */
+ void freeSlot(AllocationID allocationId);
+
+ /**
+ * Timeout the task slot for the given allocation id. The timeout is identified by the given
+ * ticket to filter invalid timeouts out.
+ *
+ * @param allocationId identifying the task slot to be timed out
+ * @param ticket allowing to filter invalid timeouts out
+ */
+ void timeoutSlot(AllocationID allocationId, UUID ticket);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotNotActiveException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotNotActiveException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotNotActiveException.java
new file mode 100644
index 0000000..b0ddc5d
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotNotActiveException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.slot;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+
+/**
+ * Exception indicating that the given {@link TaskSlot} was not in state active.
+ */
+public class SlotNotActiveException extends Exception {
+
+ private static final long serialVersionUID = 4305837511564584L;
+
+ public SlotNotActiveException(JobID jobId, AllocationID allocationId) {
+ super("No active slot for job " + jobId + " with allocation id " + allocationId + '.');
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotNotFoundException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotNotFoundException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotNotFoundException.java
new file mode 100644
index 0000000..c639b16
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/SlotNotFoundException.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.slot;
+
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+
+/**
+ * Exception indicating that a {@link TaskSlot} could not be found.
+ */
+public class SlotNotFoundException extends Exception {
+
+ private static final long serialVersionUID = -883614807750137925L;
+
+ public SlotNotFoundException(AllocationID allocationId) {
+ this("Could not find slot for " + allocationId + '.');
+ }
+
+ public SlotNotFoundException(String message) {
+ super(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlot.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlot.java
new file mode 100644
index 0000000..0942772
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlot.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.slot;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.taskmanager.Task;
+import org.apache.flink.util.Preconditions;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * Container for multiple {@link Task} belonging to the same slot. A {@link TaskSlot} can be in one
+ * of the following states:
+ * <ul>
+ * <li>Free - The slot is empty and not allocated to a job</li>
+ * <li>Releasing - The slot is about to be freed after it has become empty.</li>
+ * <li>Allocated - The slot has been allocated for a job.</li>
+ * <li>Active - The slot is in active use by a job manager which is the leader of the allocating job.</li>
+ * </ul>
+ * <p>
+ * A task slot can only be allocated if it is in state free. An allocated task slot can transition
+ * to state active.
+ *<p>
+ * An active slot allows to add tasks from the respective job and with the correct allocation id.
+ * An active slot can be marked as inactive which sets the state back to allocated.
+ * <p>
+ * An allocated or active slot can only be freed if it is empty. If it is not empty, then it's state
+ * can be set to releasing indicating that it can be freed once it becomes empty.
+ */
+public class TaskSlot {
+
+ /** Index of the task slot */
+ private final int index;
+
+ /** Resource characteristics for this slot */
+ private final ResourceProfile resourceProfile;
+
+ /** Tasks running in this slot */
+ private final Map<ExecutionAttemptID, Task> tasks;
+
+ /** State of this slot */
+ private TaskSlotState state;
+
+ /** Job id to which the slot has been allocated; null if not allocated */
+ private JobID jobId;
+
+ /** Allocation id of this slot; null if not allocated */
+ private AllocationID allocationId;
+
+ TaskSlot(final int index, final ResourceProfile resourceProfile) {
+ Preconditions.checkArgument(0 <= index, "The index must be greater than 0.");
+ this.index = index;
+ this.resourceProfile = Preconditions.checkNotNull(resourceProfile);
+
+ this.tasks = new HashMap<>(4);
+ this.state = TaskSlotState.FREE;
+
+ this.jobId = null;
+ this.allocationId = null;
+ }
+
+ // ----------------------------------------------------------------------------------
+ // State accessors
+ // ----------------------------------------------------------------------------------
+
+ public int getIndex() {
+ return index;
+ }
+
+ public ResourceProfile getResourceProfile() {
+ return resourceProfile;
+ }
+
+ public JobID getJobId() {
+ return jobId;
+ }
+
+ public AllocationID getAllocationId() {
+ return allocationId;
+ }
+
+ TaskSlotState getState() {
+ return state;
+ }
+
+ public boolean isEmpty() {
+ return tasks.isEmpty();
+ }
+
+ public boolean isFree() {
+ return TaskSlotState.FREE == state;
+ }
+
+ public boolean isActive(JobID activeJobId, AllocationID activeAllocationId) {
+ Preconditions.checkNotNull(activeJobId);
+ Preconditions.checkNotNull(activeAllocationId);
+
+ return TaskSlotState.ACTIVE == state &&
+ activeJobId.equals(jobId) &&
+ activeAllocationId.equals(allocationId);
+ }
+
+ public boolean isAllocated(JobID jobIdToCheck, AllocationID allocationIDToCheck) {
+ Preconditions.checkNotNull(jobIdToCheck);
+ Preconditions.checkNotNull(allocationIDToCheck);
+
+ return jobIdToCheck.equals(jobId) && allocationIDToCheck.equals(allocationId) &&
+ (TaskSlotState.ACTIVE == state || TaskSlotState.ALLOCATED == state);
+ }
+
+ public boolean isReleasing() {
+ return TaskSlotState.RELEASING == state;
+ }
+
+ /**
+ * Get all tasks running in this task slot.
+ *
+ * @return Iterator to all currently contained tasks in this task slot.
+ */
+ public Iterator<Task> getTasks() {
+ return tasks.values().iterator();
+ }
+
+ // ----------------------------------------------------------------------------------
+ // State changing methods
+ // ----------------------------------------------------------------------------------
+
+ /**
+ * Add the given task to the task slot. This is only possible if there is not already another
+ * task with the same execution attempt id added to the task slot. In this case, the method
+ * returns true. Otherwise the task slot is left unchanged and false is returned.
+ *
+ * In case that the task slot state is not active an {@link IllegalStateException} is thrown.
+ * In case that the task's job id and allocation id don't match with the job id and allocation
+ * id for which the task slot has been allocated, an {@link IllegalArgumentException} is thrown.
+ *
+ * @param task to be added to the task slot
+ * @throws IllegalStateException if the task slot is not in state active
+ * @return true if the task was added to the task slot; otherwise false
+ */
+ public boolean add(Task task) {
+ // Check that this slot has been assigned to the job sending this task
+ Preconditions.checkArgument(task.getJobID().equals(jobId), "The task's job id does not match the " +
+ "job id for which the slot has been allocated.");
+ Preconditions.checkArgument(task.getAllocationId().equals(allocationId), "The task's allocation " +
+ "id does not match the allocation id for which the slot has been allocated.");
+ Preconditions.checkState(TaskSlotState.ACTIVE == state, "The task slot is not in state active.");
+
+ Task oldTask = tasks.put(task.getExecutionId(), task);
+
+ if (oldTask != null) {
+ tasks.put(task.getExecutionId(), oldTask);
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ /**
+ * Remove the task identified by the given execution attempt id.
+ *
+ * @param executionAttemptId identifying the task to be removed
+ * @return The removed task if there was any; otherwise null.
+ */
+ public Task remove(ExecutionAttemptID executionAttemptId) {
+ return tasks.remove(executionAttemptId);
+ }
+
+ /**
+ * Removes all tasks from this task slot.
+ */
+ public void clear() {
+ tasks.clear();
+ }
+
+ /**
+ * Allocate the task slot for the given job and allocation id. If the slot could be allocated,
+ * or is already allocated/active for the given job and allocation id, then the method returns
+ * true. Otherwise it returns false.
+ *
+ * A slot can only be allocated if it's current state is free.
+ *
+ * @param newJobId to allocate the slot for
+ * @param newAllocationId to identify the slot allocation
+ * @return True if the slot was allocated for the given job and allocation id; otherwise false
+ */
+ public boolean allocate(JobID newJobId, AllocationID newAllocationId) {
+ if (TaskSlotState.FREE == state) {
+ // sanity checks
+ Preconditions.checkState(allocationId == null);
+ Preconditions.checkState(jobId == null);
+
+ this.jobId = Preconditions.checkNotNull(newJobId);
+ this.allocationId = Preconditions.checkNotNull(newAllocationId);
+
+ state = TaskSlotState.ALLOCATED;
+
+ return true;
+ } else if (TaskSlotState.ALLOCATED == state || TaskSlotState.ACTIVE == state) {
+ Preconditions.checkNotNull(newJobId);
+ Preconditions.checkNotNull(newAllocationId);
+
+ return newJobId.equals(jobId) && newAllocationId.equals(allocationId);
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Mark this slot as active. A slot can only be marked active if it's in state allocated.
+ *
+ * The method returns true if the slot was set to active. Otherwise it returns false.
+ *
+ * @return True if the new state of the slot is active; otherwise false
+ */
+ public boolean markActive() {
+ if (TaskSlotState.ALLOCATED == state || TaskSlotState.ACTIVE == state) {
+ state = TaskSlotState.ACTIVE;
+
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Mark the slot as inactive/allocated. A slot can only be marked as inactive/allocated if it's
+ * in state allocated or active.
+ *
+ * @return True if the new state of the slot is allocated; otherwise false
+ */
+ public boolean markInactive() {
+ if (TaskSlotState.ACTIVE == state || TaskSlotState.ALLOCATED == state) {
+ state = TaskSlotState.ALLOCATED;
+
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Mark the slot as free. A slot can only marked as free if it's empty.
+ *
+ * @return True if the new state is free; otherwise false
+ */
+ public boolean markFree() {
+ if (isEmpty()) {
+ state = TaskSlotState.FREE;
+ this.jobId = null;
+ this.allocationId = null;
+
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Mark this slot as releasing. A slot can always be marked as releasing.
+ *
+ * @return True
+ */
+ public boolean markReleasing() {
+ state = TaskSlotState.RELEASING;
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotState.java
new file mode 100644
index 0000000..e3ba903
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotState.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.slot;
+
+/**
+ * Internal task slot state
+ */
+enum TaskSlotState {
+ ACTIVE, // Slot is in active use by a job manager responsible for a job
+ ALLOCATED, // Slot has been allocated for a job but not yet given to a job manager
+ RELEASING, // Slot is not empty but tasks are failed. Upon removal of all tasks, it will be released
+ FREE // Slot is free
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
new file mode 100644
index 0000000..42cb919
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
@@ -0,0 +1,682 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.slot;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.taskmanager.Task;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * Container for multiple {@link TaskSlot} instances. Additionally, it maintains multiple indices
+ * for faster access to tasks and sets of allocated slots.
+ * <p>
+ * The task slot table automatically registers timeouts for allocated slots which cannot be assigned
+ * to a job manager.
+ * <p>
+ * Before the task slot table can be used, it must be started via the {@link #start} method.
+ */
+public class TaskSlotTable implements TimeoutListener<AllocationID> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TaskSlotTable.class);
+
+ /** Timer service used to time out allocated slots */
+ private final TimerService<AllocationID> timerService;
+
+ /** The list of all task slots */
+ private final List<TaskSlot> taskSlots;
+
+ /** Mapping from allocation id to task slot */
+ private final Map<AllocationID, TaskSlot> allocationIDTaskSlotMap;
+
+ /** Mapping from execution attempt id to task and task slot */
+ private final Map<ExecutionAttemptID, TaskSlotMapping> taskSlotMappings;
+
+ /** Mapping from job id to allocated slots for a job */
+ private final Map<JobID, Set<AllocationID>> slotsPerJob;
+
+ /** Interface for slot actions, such as freeing them or timing them out */
+ private SlotActions slotActions;
+
+ /** The timeout for allocated slots */
+ private Time slotTimeout;
+
+ /** Whether the table has been started */
+ private boolean started;
+
+ public TaskSlotTable(
+ final Collection<ResourceProfile> resourceProfiles,
+ final TimerService<AllocationID> timerService) {
+
+ int numberSlots = resourceProfiles.size();
+
+ Preconditions.checkArgument(0 < numberSlots, "The number of task slots must be greater than 0.");
+
+ this.timerService = Preconditions.checkNotNull(timerService);
+
+ taskSlots = Arrays.asList(new TaskSlot[numberSlots]);
+
+ int index = 0;
+
+ // create the task slots for the given resource profiles
+ for (ResourceProfile resourceProfile: resourceProfiles) {
+ taskSlots.set(index, new TaskSlot(index, resourceProfile));
+ ++index;
+ }
+
+ allocationIDTaskSlotMap = new HashMap<>(numberSlots);
+
+ taskSlotMappings = new HashMap<>(4 * numberSlots);
+
+ slotsPerJob = new HashMap<>(4);
+
+ slotActions = null;
+ slotTimeout = null;
+ started = false;
+ }
+
+ /**
+ * Start the task slot table with the given slot actions and slot timeout value.
+ *
+ * @param initialSlotActions to use for slot actions
+ * @param initialSlotTimeout to use for slot timeouts
+ */
+ public void start(SlotActions initialSlotActions, Time initialSlotTimeout) {
+ this.slotActions = Preconditions.checkNotNull(initialSlotActions);
+ this.slotTimeout = Preconditions.checkNotNull(initialSlotTimeout);
+
+ timerService.start(this);
+
+ started = true;
+ }
+
+ /**
+ * Stop the task slot table.
+ */
+ public void stop() {
+ started = false;
+ timerService.stop();
+ slotTimeout = null;
+ slotActions = null;
+ }
+
+ // ---------------------------------------------------------------------
+ // Slot methods
+ // ---------------------------------------------------------------------
+
+ /**
+ * Allocate the slot with the given index for the given job and allocation id. Returns true if
+ * the slot could be allocated. Otherwise it returns false;
+ *
+ * @param index of the task slot to allocate
+ * @param jobId to allocate the task slot for
+ * @param allocationId identifying the allocation
+ * @return True if the task slot could be allocated; otherwise false
+ */
+ public boolean allocateSlot(int index, JobID jobId, AllocationID allocationId) {
+ checkInit();
+
+ TaskSlot taskSlot = taskSlots.get(index);
+
+ boolean result = taskSlot.allocate(jobId, allocationId);
+
+ if (result) {
+ // update the alloction id to task slot map
+ allocationIDTaskSlotMap.put(allocationId, taskSlot);
+
+ // register a timeout for this slot since it's in state allocated
+ timerService.registerTimeout(allocationId, slotTimeout.getSize(), slotTimeout.getUnit());
+
+ // add this slot to the set of job slots
+ Set<AllocationID> slots = slotsPerJob.get(jobId);
+
+ if (slots == null) {
+ slots = new HashSet<>(4);
+ slotsPerJob.put(jobId, slots);
+ }
+
+ slots.add(allocationId);
+ }
+
+ return result;
+ }
+
+ /**
+ * Marks the slot under the given allocation id as active. If the slot could not be found, then
+ * a {@link SlotNotFoundException} is thrown.
+ *
+ * @param allocationId to identify the task slot to mark as active
+ * @throws SlotNotFoundException if the slot could not be found for the given allocation id
+ * @return True if the slot could be marked active
+ */
+ public boolean markSlotActive(AllocationID allocationId) throws SlotNotFoundException {
+ checkInit();
+
+ TaskSlot taskSlot = getTaskSlot(allocationId);
+
+ if (taskSlot != null) {
+ if (taskSlot.markActive()) {
+ // unregister a potential timeout
+ timerService.unregisterTimeout(allocationId);
+
+ return true;
+ } else {
+ return false;
+ }
+ } else {
+ throw new SlotNotFoundException(allocationId);
+ }
+ }
+
+ /**
+ * Marks the slot under the given allocation id as inactive. If the slot could not be found,
+ * then a {@link SlotNotFoundException} is thrown.
+ *
+ * @param allocationId to identify the task slot to mark as inactive
+ * @throws SlotNotFoundException if the slot could not be found for the given allocation id
+ * @return True if the slot could be marked inactive
+ */
+ public boolean markSlotInactive(AllocationID allocationId) throws SlotNotFoundException {
+ checkInit();
+
+ TaskSlot taskSlot = getTaskSlot(allocationId);
+
+ if (taskSlot != null) {
+ if (taskSlot.markInactive()) {
+ // register a timeout to free the slot
+ timerService.registerTimeout(allocationId, slotTimeout.getSize(), slotTimeout.getUnit());
+
+ return true;
+ } else {
+ return false;
+ }
+ } else {
+ throw new SlotNotFoundException(allocationId);
+ }
+ }
+
+ /**
+ * Try to free the slot. If the slot is empty it will set the state of the task slot to free
+ * and return its index. If the slot is not empty, then it will set the state of the task slot
+ * to releasing, fail all tasks and return -1.
+ *
+ * @param allocationId identifying the task slot to be freed
+ * @throws SlotNotFoundException if there is not task slot for the given allocation id
+ * @return Index of the freed slot if the slot could be freed; otherwise -1
+ */
+ public int freeSlot(AllocationID allocationId) throws SlotNotFoundException {
+ return freeSlot(allocationId, new Exception("The task slot of this task is being freed."));
+ }
+
+ /**
+ * Tries to free the slot. If the slot is empty it will set the state of the task slot to free
+ * and return its index. If the slot is not empty, then it will set the state of the task slot
+ * to releasing, fail all tasks and return -1.
+ *
+ * @param allocationId identifying the task slot to be freed
+ * @param cause to fail the tasks with if slot is not empty
+ * @throws SlotNotFoundException if there is not task slot for the given allocation id
+ * @return Index of the freed slot if the slot could be freed; otherwise -1
+ */
+ public int freeSlot(AllocationID allocationId, Throwable cause) throws SlotNotFoundException {
+ checkInit();
+
+ TaskSlot taskSlot = getTaskSlot(allocationId);
+
+ if (taskSlot != null) {
+ LOG.info("Free slot {}.", allocationId, cause);
+
+ final JobID jobId = taskSlot.getJobId();
+
+ if (taskSlot.markFree()) {
+ // remove the allocation id to task slot mapping
+ allocationIDTaskSlotMap.remove(allocationId);
+
+ // unregister a potential timeout
+ timerService.unregisterTimeout(allocationId);
+
+ Set<AllocationID> slots = slotsPerJob.get(jobId);
+
+ if (slots == null) {
+ throw new IllegalStateException("There are no more slots allocated for the job " + jobId +
+ ". This indicates a programming bug.");
+ }
+
+ slots.remove(allocationId);
+
+ if (slots.isEmpty()) {
+ slotsPerJob.remove(jobId);
+ }
+
+ return taskSlot.getIndex();
+ } else {
+ // we couldn't free the task slot because it still contains task, fail the tasks
+ // and set the slot state to releasing so that it gets eventually freed
+ taskSlot.markReleasing();
+
+ Iterator<Task> taskIterator = taskSlot.getTasks();
+
+ while (taskIterator.hasNext()) {
+ taskIterator.next().failExternally(cause);
+ }
+
+ return -1;
+ }
+ } else {
+ throw new SlotNotFoundException(allocationId);
+ }
+ }
+
+ /**
+ * Check whether the timeout with ticket is valid for the given allocation id.
+ *
+ * @param allocationId to check against
+ * @param ticket of the timeout
+ * @return True if the timeout is valid; otherwise false
+ */
+ public boolean isValidTimeout(AllocationID allocationId, UUID ticket) {
+ checkInit();
+
+ return timerService.isValid(allocationId, ticket);
+ }
+
+ /**
+ * Check whether the slot for the given index is allocated for the given job and allocation id.
+ *
+ * @param index of the task slot
+ * @param jobId for which the task slot should be allocated
+ * @param allocationId which should match the task slot's allocation id
+ * @return True if the given task slot is allocated for the given job and allocation id
+ */
+ public boolean isAllocated(int index, JobID jobId, AllocationID allocationId) {
+ checkInit();
+
+ TaskSlot taskSlot = taskSlots.get(index);
+
+ return taskSlot.isAllocated(jobId, allocationId);
+ }
+
+ /**
+ * Check whether there exists an active slot for the given job and allocation id.
+ *
+ * @param jobId of the allocated slot
+ * @param allocationId identifying the allocation
+ * @return True if there exists a task slot which is active for the given job and allocation id.
+ */
+ public boolean existActiveSlot(JobID jobId, AllocationID allocationId) {
+ TaskSlot taskSlot = getTaskSlot(allocationId);
+
+ if (taskSlot != null) {
+ return taskSlot.isActive(jobId, allocationId);
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Check whether the task slot with the given index is free.
+ *
+ * @param index of the task slot
+ * @return True if the task slot is free; otherwise false
+ */
+ public boolean isSlotFree(int index) {
+ TaskSlot taskSlot = taskSlots.get(index);
+
+ return taskSlot.isFree();
+ }
+
+ /**
+ * Check whether the job has allocated (not active) slots.
+ *
+ * @param jobId for which to check for allocated slots
+ * @return True if there are allocated slots for the given job id.
+ */
+ public boolean hasAllocatedSlots(JobID jobId) {
+ return getAllocatedSlots(jobId).hasNext();
+ }
+
+ /**
+ * Return an iterator of allocated slots (their allocation ids) for the given job id.
+ *
+ * @param jobId for which to return the allocated slots
+ * @return Iterator of allocation ids of allocated slots.
+ */
+ public Iterator<AllocationID> getAllocatedSlots(JobID jobId) {
+ return new AllocationIDIterator(jobId, TaskSlotState.ALLOCATED);
+ }
+
+ /**
+ * Return an iterator of active slots (their application ids) for the given job id.
+ *
+ * @param jobId for which to return the active slots
+ * @return Iterator of allocation ids of active slots
+ */
+ public Iterator<AllocationID> getActiveSlots(JobID jobId) {
+ return new AllocationIDIterator(jobId, TaskSlotState.ACTIVE);
+ }
+
+ // ---------------------------------------------------------------------
+ // Task methods
+ // ---------------------------------------------------------------------
+
+ /**
+ * Add the given task to the slot identified by the task's allocation id.
+ *
+ * @param task to add to the task slot with the respective allocation id
+ * @throws SlotNotFoundException if there was no slot for the given allocation id
+ * @throws SlotNotActiveException if there was no slot active for task's job and allocation id
+ * @return True if the task could be added to the task slot; otherwise false
+ */
+ public boolean addTask(Task task) throws SlotNotFoundException, SlotNotActiveException {
+ Preconditions.checkNotNull(task);
+
+ TaskSlot taskSlot = getTaskSlot(task.getAllocationId());
+
+ if (taskSlot != null) {
+ if (taskSlot.isActive(task.getJobID(), task.getAllocationId())) {
+ if (taskSlot.add(task)) {
+ taskSlotMappings.put(task.getExecutionId(), new TaskSlotMapping(task, taskSlot));
+
+ return true;
+ } else {
+ return false;
+ }
+ } else {
+ throw new SlotNotActiveException(task.getJobID(), task.getAllocationId());
+ }
+ } else {
+ throw new SlotNotFoundException(taskSlot.getAllocationId());
+ }
+ }
+
+ /**
+ * Remove the task with the given execution attempt id from its task slot. If the owning task
+ * slot is in state releasing and empty after removing the task, the slot is freed via the
+ * slot actions.
+ *
+ * @param executionAttemptID identifying the task to remove
+ * @return The removed task if there is any for the given execution attempt id; otherwise null
+ */
+ public Task removeTask(ExecutionAttemptID executionAttemptID) {
+ TaskSlotMapping taskSlotMapping = taskSlotMappings.remove(executionAttemptID);
+
+ if (taskSlotMapping != null) {
+ Task task = taskSlotMapping.getTask();
+ TaskSlot taskSlot = taskSlotMapping.getTaskSlot();
+
+ taskSlot.remove(task.getExecutionId());
+
+ if (taskSlot.isReleasing() && taskSlot.isEmpty()) {
+ slotActions.freeSlot(taskSlot.getAllocationId());
+ }
+
+ return task;
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Get the task for the given execution attempt id. If none could be found, then return null.
+ *
+ * @param executionAttemptID identifying the requested task
+ * @return The task for the given execution attempt id if it exist; otherwise null
+ */
+ public Task getTask(ExecutionAttemptID executionAttemptID) {
+ TaskSlotMapping taskSlotMapping = taskSlotMappings.get(executionAttemptID);
+
+ if (taskSlotMapping != null) {
+ return taskSlotMapping.getTask();
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Return an iterator over all tasks for a given job.
+ *
+ * @param jobId identifying the job of the requested tasks
+ * @return Iterator over all task for a given job
+ */
+ public Iterator<Task> getTasks(JobID jobId) {
+ return new TaskIterator(jobId);
+ }
+
+ // ---------------------------------------------------------------------
+ // TimeoutListener methods
+ // ---------------------------------------------------------------------
+
+ @Override
+ public void notifyTimeout(AllocationID key, UUID ticket) {
+ if (slotActions != null) {
+ slotActions.timeoutSlot(key, ticket);
+ }
+ }
+
+ // ---------------------------------------------------------------------
+ // Internal methods
+ // ---------------------------------------------------------------------
+
+ private TaskSlot getTaskSlot(AllocationID allocationId) {
+ Preconditions.checkNotNull(allocationId);
+
+ TaskSlot taskSlot = allocationIDTaskSlotMap.get(allocationId);
+
+ return taskSlot;
+ }
+
+ private void checkInit() {
+ Preconditions.checkState(started, "The " + TaskSlotTable.class.getSimpleName() + " has to be started.");
+ }
+
+ // ---------------------------------------------------------------------
+ // Static utility classes
+ // ---------------------------------------------------------------------
+
+ /**
+ * Mapping class between a {@link Task} and a {@link TaskSlot}.
+ */
+ private static final class TaskSlotMapping {
+ private final Task task;
+ private final TaskSlot taskSlot;
+
+
+ private TaskSlotMapping(Task task, TaskSlot taskSlot) {
+ this.task = Preconditions.checkNotNull(task);
+ this.taskSlot = Preconditions.checkNotNull(taskSlot);
+ }
+
+ public Task getTask() {
+ return task;
+ }
+
+ public TaskSlot getTaskSlot() {
+ return taskSlot;
+ }
+ }
+
+ /**
+ * Iterator over {@link AllocationID} of the {@link TaskSlot} of a given job. Additionally,
+ * the task slots identified by the allocation ids are in the given state.
+ */
+ private final class AllocationIDIterator implements Iterator<AllocationID> {
+ private final Iterator<TaskSlot> iterator;
+
+ private AllocationIDIterator(JobID jobId, TaskSlotState state) {
+ iterator = new TaskSlotIterator(jobId, state);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public AllocationID next() {
+ try {
+ return iterator.next().getAllocationId();
+ } catch (NoSuchElementException e) {
+ throw new NoSuchElementException("No more allocation ids.");
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("Cannot remove allocation ids via this iterator.");
+ }
+ }
+
+ /**
+ * Iterator over {@link TaskSlot} which fulfill a given state condition and belong to the given
+ * job.
+ */
+ private final class TaskSlotIterator implements Iterator<TaskSlot> {
+ private final Iterator<AllocationID> allSlots;
+ private final TaskSlotState state;
+
+ private TaskSlot currentSlot;
+
+ private TaskSlotIterator(JobID jobId, TaskSlotState state) {
+
+ Set<AllocationID> allocationIds = slotsPerJob.get(jobId);
+
+ if (allocationIds == null || allocationIds.isEmpty()) {
+ allSlots = Collections.emptyIterator();
+ } else {
+ allSlots = allocationIds.iterator();
+ }
+
+ this.state = Preconditions.checkNotNull(state);
+
+ this.currentSlot = null;
+ }
+
+ @Override
+ public boolean hasNext() {
+ while (currentSlot == null && allSlots.hasNext()) {
+ AllocationID tempSlot = allSlots.next();
+
+ TaskSlot taskSlot = getTaskSlot(tempSlot);
+
+ if (taskSlot != null && taskSlot.getState() == state) {
+ currentSlot = taskSlot;
+ }
+ }
+
+ return currentSlot != null;
+ }
+
+ @Override
+ public TaskSlot next() {
+ if (currentSlot != null) {
+ TaskSlot result = currentSlot;
+
+ currentSlot = null;
+
+ return result;
+ } else {
+ while (true) {
+ AllocationID tempSlot;
+
+ try {
+ tempSlot = allSlots.next();
+ } catch (NoSuchElementException e) {
+ throw new NoSuchElementException("No more task slots.");
+ }
+
+ TaskSlot taskSlot = getTaskSlot(tempSlot);
+
+ if (taskSlot != null && taskSlot.getState() == state) {
+ return taskSlot;
+ }
+ }
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("Cannot remove task slots via this iterator.");
+ }
+ }
+
+ /**
+ * Iterator over all {@link Task} for a given job
+ */
+ private final class TaskIterator implements Iterator<Task> {
+ private final Iterator<TaskSlot> taskSlotIterator;
+
+ private Iterator<Task> currentTasks;
+
+ private TaskIterator(JobID jobId) {
+ this.taskSlotIterator = new TaskSlotIterator(jobId, TaskSlotState.ACTIVE);
+
+ this.currentTasks = null;
+ }
+
+ @Override
+ public boolean hasNext() {
+ while ((currentTasks == null || !currentTasks.hasNext()) && taskSlotIterator.hasNext()) {
+ TaskSlot taskSlot = taskSlotIterator.next();
+
+ currentTasks = taskSlot.getTasks();
+ }
+
+ return (currentTasks != null && currentTasks.hasNext());
+ }
+
+ @Override
+ public Task next() {
+ while ((currentTasks == null || !currentTasks.hasNext())) {
+ TaskSlot taskSlot;
+
+ try {
+ taskSlot = taskSlotIterator.next();
+ } catch (NoSuchElementException e) {
+ throw new NoSuchElementException("No more tasks.");
+ }
+
+ currentTasks = taskSlot.getTasks();
+ }
+
+ return currentTasks.next();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("Cannot remove tasks via this iterator.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TimeoutListener.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TimeoutListener.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TimeoutListener.java
new file mode 100644
index 0000000..3e75d74
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TimeoutListener.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.slot;
+
+import java.util.UUID;
+
+/**
+ * Listener for timeout events by the {@link TimerService}.
+ * @param <K> Type of the timeout key
+ */
+public interface TimeoutListener<K> {
+
+ /**
+ * Notify the listener about the timeout for an event identified by key. Additionally the method
+ * is called with the timeout ticket which allows to identify outdated timeout events.
+ *
+ * @param key identifying the timed out event
+ * @param ticket used to check whether the timeout is still valid
+ */
+ void notifyTimeout(K key, UUID ticket);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TimerService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TimerService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TimerService.java
new file mode 100644
index 0000000..e28e801
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TimerService.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.slot;
+
+import org.apache.flink.util.Preconditions;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Service to register timeouts for a given key. The timeouts are identified by a ticket so that
+ * newly registered timeouts for the same key can be distinguished from older timeouts.
+ *
+ * @param <K> Type of the key
+ */
+public class TimerService<K> {
+
+ /** Executor service for the scheduled timeouts */
+ private final ScheduledExecutorService scheduledExecutorService;
+
+ /** Map of currently active timeouts */
+ private final Map<K, Timeout<K>> timeouts;
+
+ /** Listener which is notified about occurring timeouts */
+ private TimeoutListener<K> timeoutListener;
+
+ public TimerService(final ScheduledExecutorService scheduledExecutorService) {
+ this.scheduledExecutorService = Preconditions.checkNotNull(scheduledExecutorService);
+
+ this.timeouts = new HashMap<>(16);
+ this.timeoutListener = null;
+ }
+
+ public void start(TimeoutListener<K> initialTimeoutListener) {
+ // sanity check; We only allow to assign a timeout listener once
+ Preconditions.checkState(!scheduledExecutorService.isShutdown());
+ Preconditions.checkState(timeoutListener == null);
+
+ this.timeoutListener = Preconditions.checkNotNull(initialTimeoutListener);
+ }
+
+ public void stop() {
+ for (K key: timeouts.keySet()) {
+ unregisterTimeout(key);
+ }
+
+ timeoutListener = null;
+
+ scheduledExecutorService.shutdown();
+ }
+
+ /**
+ * Register a timeout for the given key which shall occur in the given delay.
+ *
+ * @param key for which to register the timeout
+ * @param delay until the timeout
+ * @param unit of the timeout delay
+ */
+ public void registerTimeout(final K key, final long delay, final TimeUnit unit) {
+ Preconditions.checkState(timeoutListener != null, "The " + getClass().getSimpleName() +
+ " has not been started.");
+
+ if (timeouts.containsKey(key)) {
+ unregisterTimeout(key);
+ }
+
+ timeouts.put(key, new Timeout<>(timeoutListener, key, delay, unit, scheduledExecutorService));
+ }
+
+ /**
+ * Unregister the timeout for the given key.
+ *
+ * @param key for which to unregister the timeout
+ */
+ public void unregisterTimeout(K key) {
+ Timeout<K> timeout = timeouts.remove(key);
+
+ if (timeout != null) {
+ timeout.cancel();
+ }
+ }
+
+ /**
+ * Check whether the timeout for the given key and ticket is still valid (not yet unregistered
+ * and not yet overwritten).
+ *
+ * @param key for which to check the timeout
+ * @param ticket of the timeout
+ * @return True if the timeout ticket is still valid; otherwise false
+ */
+ public boolean isValid(K key, UUID ticket) {
+ if (timeouts.containsKey(key)) {
+ Timeout<K> timeout = timeouts.get(key);
+
+ return timeout.getTicket().equals(ticket);
+ } else {
+ return false;
+ }
+ }
+
+ // ---------------------------------------------------------------------
+ // Static utility classes
+ // ---------------------------------------------------------------------
+
+ private static final class Timeout<K> implements Runnable {
+
+ private final TimeoutListener<K> timeoutListener;
+ private final K key;
+ private final ScheduledFuture<?> scheduledTimeout;
+ private final UUID ticket;
+
+ Timeout(
+ final TimeoutListener<K> timeoutListener,
+ final K key,
+ final long delay,
+ final TimeUnit unit,
+ final ScheduledExecutorService scheduledExecutorService) {
+
+ Preconditions.checkNotNull(scheduledExecutorService);
+
+ this.timeoutListener = Preconditions.checkNotNull(timeoutListener);
+ this.key = Preconditions.checkNotNull(key);
+ this.scheduledTimeout = scheduledExecutorService.schedule(this, delay, unit);
+ this.ticket = UUID.randomUUID();
+ }
+
+ UUID getTicket() {
+ return ticket;
+ }
+
+ void cancel() {
+ scheduledTimeout.cancel(true);
+ }
+
+ @Override
+ public void run() {
+ timeoutListener.notifyTimeout(key, ticket);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index 0bd3a89..096bcef 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -133,7 +133,7 @@ public class Task implements Runnable, TaskActions {
private final ExecutionAttemptID executionId;
/** ID which identifies the slot in which the task is supposed to run */
- private final AllocationID allocationID;
+ private final AllocationID allocationId;
/** TaskInfo object for this task */
private final TaskInfo taskInfo;
@@ -273,7 +273,7 @@ public class Task implements Runnable, TaskActions {
this.jobId = checkNotNull(tdd.getJobID());
this.vertexId = checkNotNull(tdd.getVertexID());
this.executionId = checkNotNull(tdd.getExecutionId());
- this.allocationID = checkNotNull(tdd.getAllocationID());
+ this.allocationId = checkNotNull(tdd.getAllocationID());
this.taskNameWithSubtask = taskInfo.getTaskNameWithSubtasks();
this.jobConfiguration = checkNotNull(tdd.getJobConfiguration());
this.taskConfiguration = checkNotNull(tdd.getTaskConfiguration());
@@ -387,8 +387,8 @@ public class Task implements Runnable, TaskActions {
return executionId;
}
- public AllocationID getAllocationID() {
- return allocationID;
+ public AllocationID getAllocationId() {
+ return allocationId;
}
public TaskInfo getTaskInfo() {
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartialConsumePipelinedResultTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartialConsumePipelinedResultTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartialConsumePipelinedResultTest.java
index af8aa69..97f42b1 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartialConsumePipelinedResultTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartialConsumePipelinedResultTest.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.io.network.partition;
-import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.io.network.api.reader.BufferReader;
http://git-wip-us.apache.org/repos/asf/flink/blob/a122bfd8/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index ecbd9b5..baae251 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -39,6 +39,7 @@ import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequ
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.TestingSerialRpcService;
+import org.apache.flink.runtime.taskexecutor.slot.TaskSlotTable;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.util.TestLogger;
@@ -83,6 +84,7 @@ public class TaskExecutorTest extends TestLogger {
mock(TaskManagerMetricGroup.class),
mock(BroadcastVariableManager.class),
mock(FileCache.class),
+ mock(TaskSlotTable.class),
mock(FatalErrorHandler.class));
taskManager.start();
@@ -139,6 +141,7 @@ public class TaskExecutorTest extends TestLogger {
mock(TaskManagerMetricGroup.class),
mock(BroadcastVariableManager.class),
mock(FileCache.class),
+ mock(TaskSlotTable.class),
mock(FatalErrorHandler.class));
taskManager.start();
@@ -211,6 +214,7 @@ public class TaskExecutorTest extends TestLogger {
mock(TaskManagerMetricGroup.class),
mock(BroadcastVariableManager.class),
mock(FileCache.class),
+ mock(TaskSlotTable.class),
mock(FatalErrorHandler.class));
taskManager.start();
[29/50] [abbrv] flink git commit: [FLINK-4375] [distributed
coordination] Implement new JobManager creation, initialization,
and basic RPC methods
Posted by tr...@apache.org.
[FLINK-4375] [distributed coordination] Implement new JobManager creation, initialization, and basic RPC methods
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5f62d3b7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5f62d3b7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5f62d3b7
Branch: refs/heads/flip-6
Commit: 5f62d3b707d3b18a09c19db316366473a226d2ac
Parents: 94a0646
Author: Kurt Young <yk...@gmail.com>
Authored: Wed Oct 12 23:25:16 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:31 2016 +0100
----------------------------------------------------------------------
.../org/apache/flink/util/ExceptionUtils.java | 20 +
.../java/org/apache/flink/util/StringUtils.java | 14 +
.../apache/flink/runtime/blob/BlobServer.java | 65 +-
.../apache/flink/runtime/blob/BlobStore.java | 3 +-
.../apache/flink/runtime/blob/BlobUtils.java | 8 +-
.../flink/runtime/blob/FileSystemBlobStore.java | 53 +-
.../flink/runtime/blob/VoidBlobStore.java | 2 +-
.../CheckpointCoordinatorGateway.java | 2 +-
.../HighAvailabilityServices.java | 13 +
.../runtime/highavailability/NonHaServices.java | 13 +
.../highavailability/RunningJobsRegistry.java | 66 ++
.../highavailability/ZookeeperHaServices.java | 115 ++-
.../highavailability/nonha/NonHaRegistry.java | 62 ++
.../runtime/jobmanager/OnCompletionActions.java | 3 +-
.../runtime/jobmaster/JobManagerRunner.java | 269 ++++--
.../runtime/jobmaster/JobManagerServices.java | 86 +-
.../flink/runtime/jobmaster/JobMaster.java | 900 +++++++------------
.../runtime/jobmaster/JobMasterGateway.java | 115 +--
.../jobmaster/MiniClusterJobDispatcher.java | 61 +-
.../message/DisposeSavepointResponse.java | 49 -
.../message/TriggerSavepointResponse.java | 74 --
.../apache/flink/runtime/rpc/RpcService.java | 4 +-
.../taskexecutor/JobManagerConnection.java | 25 +-
.../runtime/taskexecutor/TaskExecutor.java | 107 ++-
.../rpc/RpcCheckpointResponder.java | 2 +-
.../taskexecutor/rpc/RpcInputSplitProvider.java | 8 +-
.../rpc/RpcPartitionStateChecker.java | 8 +-
.../RpcResultPartitionConsumableNotifier.java | 7 +-
.../apache/flink/runtime/akka/AkkaUtils.scala | 6 +
.../TestingHighAvailabilityServices.java | 14 +
.../jobmaster/JobManagerRunnerMockTest.java | 58 +-
.../runtime/jobmaster/JobManagerRunnerTest.java | 24 +
.../runtime/operators/DataSinkTaskTest.java | 2 +-
33 files changed, 1252 insertions(+), 1006 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java b/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java
index 7227006..0f6f24f 100644
--- a/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java
+++ b/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java
@@ -99,6 +99,26 @@ public final class ExceptionUtils {
}
/**
+ * Throws the given {@code Throwable} in scenarios where the signatures do allow to
+ * throw a Exception. Errors and Exceptions are thrown directly, other "exotic"
+ * subclasses of Throwable are wrapped in an Exception.
+ *
+ * @param t The throwable to be thrown.
+ * @param parentMessage The message for the parent Exception, if one is needed.
+ */
+ public static void rethrowException(Throwable t, String parentMessage) throws Exception {
+ if (t instanceof Error) {
+ throw (Error) t;
+ }
+ else if (t instanceof Exception) {
+ throw (Exception) t;
+ }
+ else {
+ throw new Exception(parentMessage, t);
+ }
+ }
+
+ /**
* Tries to throw the given {@code Throwable} in scenarios where the signatures allows only IOExceptions
* (and RuntimeException and Error). Throws this exception directly, if it is an IOException,
* a RuntimeException, or an Error. Otherwise does nothing.
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/StringUtils.java b/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
index 10b6304..3c32d77 100644
--- a/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
+++ b/flink-core/src/main/java/org/apache/flink/util/StringUtils.java
@@ -335,4 +335,18 @@ public final class StringUtils {
return null;
}
}
+
+ public static boolean isNullOrWhitespaceOnly(String str) {
+ if (str == null || str.length() == 0) {
+ return true;
+ }
+
+ final int len = str.length();
+ for (int i = 0; i < len; i++) {
+ if (!Character.isWhitespace(str.charAt(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java
index b800500..33f9db7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java
@@ -22,7 +22,11 @@ import org.apache.commons.io.FileUtils;
import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.HighAvailabilityOptions;
import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.jobmanager.HighAvailabilityMode;
import org.apache.flink.runtime.net.SSLUtils;
import org.apache.flink.util.NetUtils;
@@ -45,6 +49,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.StringUtils.isNullOrWhitespaceOnly;
/**
* This class implements the BLOB server. The BLOB server is responsible for listening for incoming requests and
@@ -96,9 +101,16 @@ public class BlobServer extends Thread implements BlobService {
* thrown if the BLOB server cannot bind to a free network port
*/
public BlobServer(Configuration config) throws IOException {
- checkNotNull(config, "Configuration");
+ this(config, createBlobStoreFromConfig(config));
+ }
- HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(config);
+ public BlobServer(Configuration config, HighAvailabilityServices haServices) throws IOException {
+ this(config, haServices.createBlobStore());
+ }
+
+ private BlobServer(Configuration config, BlobStore blobStore) throws IOException {
+ checkNotNull(config);
+ this.blobStore = checkNotNull(blobStore);
this.blobServiceConfiguration = config;
@@ -107,14 +119,6 @@ public class BlobServer extends Thread implements BlobService {
this.storageDir = BlobUtils.initStorageDirectory(storageDirectory);
LOG.info("Created BLOB server storage directory {}", storageDir);
- if (highAvailabilityMode == HighAvailabilityMode.NONE) {
- this.blobStore = new VoidBlobStore();
- } else if (highAvailabilityMode == HighAvailabilityMode.ZOOKEEPER) {
- this.blobStore = new FileSystemBlobStore(config);
- } else {
- throw new IllegalConfigurationException("Unexpected high availability mode '" + highAvailabilityMode + ".");
- }
-
// configure the maximum number of concurrent connections
final int maxConnections = config.getInteger(
ConfigConstants.BLOB_FETCH_CONCURRENT_KEY, ConfigConstants.DEFAULT_BLOB_FETCH_CONCURRENT);
@@ -135,13 +139,7 @@ public class BlobServer extends Thread implements BlobService {
backlog = ConfigConstants.DEFAULT_BLOB_FETCH_BACKLOG;
}
- if (highAvailabilityMode == HighAvailabilityMode.NONE) {
- // Add shutdown hook to delete storage directory
- this.shutdownHook = BlobUtils.addShutdownHook(this, LOG);
- }
- else {
- this.shutdownHook = null;
- }
+ this.shutdownHook = BlobUtils.addShutdownHook(this, LOG);
if (config.getBoolean(ConfigConstants.BLOB_SERVICE_SSL_ENABLED,
ConfigConstants.DEFAULT_BLOB_SERVICE_SSL_ENABLED)) {
@@ -451,4 +449,37 @@ public class BlobServer extends Thread implements BlobService {
}
}
+ private static BlobStore createBlobStoreFromConfig(Configuration config) throws IOException {
+ HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(config);
+
+ if (highAvailabilityMode == HighAvailabilityMode.NONE) {
+ return new VoidBlobStore();
+ } else if (highAvailabilityMode == HighAvailabilityMode.ZOOKEEPER) {
+ final String storagePath = config.getValue(HighAvailabilityOptions.HA_STORAGE_PATH);
+ if (isNullOrWhitespaceOnly(storagePath)) {
+ throw new IllegalConfigurationException("Configuration is missing the mandatory parameter: " +
+ HighAvailabilityOptions.HA_STORAGE_PATH);
+ }
+
+ final Path path;
+ try {
+ path = new Path(storagePath);
+ } catch (Exception e) {
+ throw new IOException("Invalid path for highly available storage (" +
+ HighAvailabilityOptions.HA_STORAGE_PATH.key() + ')', e);
+ }
+
+ final FileSystem fileSystem;
+ try {
+ fileSystem = path.getFileSystem();
+ } catch (Exception e) {
+ throw new IOException("Could not create FileSystem for highly available storage (" +
+ HighAvailabilityOptions.HA_STORAGE_PATH.key() + ')', e);
+ }
+
+ return new FileSystemBlobStore(fileSystem, storagePath);
+ } else {
+ throw new IllegalConfigurationException("Unexpected high availability mode '" + highAvailabilityMode + ".");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobStore.java
index 1e72d91..7050338 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobStore.java
@@ -25,7 +25,7 @@ import java.io.File;
/**
* A blob store.
*/
-interface BlobStore {
+public interface BlobStore {
/**
* Copies the local file to the blob store.
@@ -93,5 +93,4 @@ interface BlobStore {
* Cleans up the store and deletes all blobs.
*/
void cleanUp();
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java
index e74fa6f..136df09 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobUtils.java
@@ -24,6 +24,7 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
import org.apache.flink.util.IOUtils;
+import org.apache.flink.util.StringUtils;
import org.slf4j.Logger;
import java.io.EOFException;
@@ -73,7 +74,7 @@ public class BlobUtils {
*/
static File initStorageDirectory(String storageDirectory) {
File baseDir;
- if (storageDirectory == null || storageDirectory.trim().isEmpty()) {
+ if (StringUtils.isNullOrWhitespaceOnly(storageDirectory)) {
baseDir = new File(System.getProperty("java.io.tmpdir"));
}
else {
@@ -81,10 +82,9 @@ public class BlobUtils {
}
File storageDir;
- final int MAX_ATTEMPTS = 10;
- int attempt;
- for(attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+ final int MAX_ATTEMPTS = 10;
+ for(int attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
storageDir = new File(baseDir, String.format(
"blobStore-%s", UUID.randomUUID().toString()));
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java
index deba738..2c05002 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/FileSystemBlobStore.java
@@ -20,12 +20,7 @@ package org.apache.flink.runtime.blob;
import com.google.common.io.Files;
-import org.apache.commons.lang3.StringUtils;
-
import org.apache.flink.api.common.JobID;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.HighAvailabilityOptions;
-import org.apache.flink.configuration.IllegalConfigurationException;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
import org.apache.flink.util.IOUtils;
@@ -38,7 +33,6 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.net.URI;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -47,25 +41,24 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
*
* <p>This is used in addition to the local blob storage for high availability.
*/
-class FileSystemBlobStore implements BlobStore {
+public class FileSystemBlobStore implements BlobStore {
private static final Logger LOG = LoggerFactory.getLogger(FileSystemBlobStore.class);
+ /** The file system in which blobs are stored */
+ private final FileSystem fileSystem;
+
/** The base path of the blob store */
private final String basePath;
- FileSystemBlobStore(Configuration config) throws IOException {
- String storagePath = config.getValue(HighAvailabilityOptions.HA_STORAGE_PATH);
-
- if (storagePath == null || StringUtils.isBlank(storagePath)) {
- throw new IllegalConfigurationException("Missing high-availability storage path for metadata." +
- " Specify via configuration key '" + HighAvailabilityOptions.HA_STORAGE_PATH + "'.");
- }
+ public FileSystemBlobStore(FileSystem fileSystem, String storagePath) throws IOException {
+ this.fileSystem = checkNotNull(fileSystem);
+ this.basePath = checkNotNull(storagePath) + "/blob";
- this.basePath = storagePath + "/blob";
+ LOG.info("Creating highly available BLOB storage directory at {}", basePath);
- FileSystem.get(new Path(basePath).toUri()).mkdirs(new Path(basePath));
- LOG.info("Created blob directory {}.", basePath);
+ fileSystem.mkdirs(new Path(basePath));
+ LOG.debug("Created highly available BLOB storage directory at {}", basePath);
}
// - Put ------------------------------------------------------------------
@@ -81,9 +74,7 @@ class FileSystemBlobStore implements BlobStore {
}
private void put(File fromFile, String toBlobPath) throws Exception {
- try (OutputStream os = FileSystem.get(new URI(toBlobPath))
- .create(new Path(toBlobPath), true)) {
-
+ try (OutputStream os = fileSystem.create(new Path(toBlobPath), true)) {
LOG.debug("Copying from {} to {}.", fromFile, toBlobPath);
Files.copy(fromFile, os);
}
@@ -106,16 +97,15 @@ class FileSystemBlobStore implements BlobStore {
checkNotNull(toFile, "File");
if (!toFile.exists() && !toFile.createNewFile()) {
- throw new IllegalStateException("Failed to create target file to copy to");
+ throw new IOException("Failed to create target file to copy to");
}
- final URI fromUri = new URI(fromBlobPath);
final Path fromPath = new Path(fromBlobPath);
- if (FileSystem.get(fromUri).exists(fromPath)) {
- try (InputStream is = FileSystem.get(fromUri).open(fromPath)) {
- FileOutputStream fos = new FileOutputStream(toFile);
-
+ if (fileSystem.exists(fromPath)) {
+ try (InputStream is = fileSystem.open(fromPath);
+ FileOutputStream fos = new FileOutputStream(toFile))
+ {
LOG.debug("Copying from {} to {}.", fromBlobPath, toFile);
IOUtils.copyBytes(is, fos); // closes the streams
}
@@ -145,17 +135,16 @@ class FileSystemBlobStore implements BlobStore {
private void delete(String blobPath) {
try {
LOG.debug("Deleting {}.", blobPath);
-
- FileSystem fs = FileSystem.get(new URI(blobPath));
+
Path path = new Path(blobPath);
- fs.delete(path, true);
+ fileSystem.delete(path, true);
// send a call to delete the directory containing the file. This will
// fail (and be ignored) when some files still exist.
try {
- fs.delete(path.getParent(), false);
- fs.delete(new Path(basePath), false);
+ fileSystem.delete(path.getParent(), false);
+ fileSystem.delete(new Path(basePath), false);
} catch (IOException ignored) {}
}
catch (Exception e) {
@@ -168,7 +157,7 @@ class FileSystemBlobStore implements BlobStore {
try {
LOG.debug("Cleaning up {}.", basePath);
- FileSystem.get(new URI(basePath)).delete(new Path(basePath), true);
+ fileSystem.delete(new Path(basePath), true);
}
catch (Exception e) {
LOG.error("Failed to clean up recovery directory.");
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/blob/VoidBlobStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/VoidBlobStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/VoidBlobStore.java
index 1b71add..ece2ac1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/VoidBlobStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/VoidBlobStore.java
@@ -25,7 +25,7 @@ import java.io.File;
/**
* A blob store doing nothing.
*/
-class VoidBlobStore implements BlobStore {
+public class VoidBlobStore implements BlobStore {
@Override
public void put(File localFile, BlobKey blobKey) throws Exception {
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java
index 5a01e4d..2634006 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java
@@ -34,5 +34,5 @@ public interface CheckpointCoordinatorGateway extends RpcGateway {
void declineCheckpoint(
final JobID jobID,
final ExecutionAttemptID executionAttemptID,
- final CheckpointMetaData checkpoint);
+ final long checkpointID);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
index a26886a..5d78ffc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
@@ -19,11 +19,14 @@
package org.apache.flink.runtime.highavailability;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.blob.BlobStore;
import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import java.io.IOException;
+
/**
* This class gives access to all services needed for
*
@@ -72,4 +75,14 @@ public interface HighAvailabilityServices {
* Gets the submitted job graph store for the job manager
*/
SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception;
+
+ /**
+ * Gets the registry that holds information about whether jobs are currently running.
+ */
+ RunningJobsRegistry getRunningJobsRegistry() throws Exception;
+
+ /**
+ * Creates the BLOB store in which BLOBs are stored in a highly-available fashion.
+ */
+ BlobStore createBlobStore() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
index 2c6295c..d7fd2bf 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
@@ -19,8 +19,11 @@
package org.apache.flink.runtime.highavailability;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.blob.BlobStore;
+import org.apache.flink.runtime.blob.VoidBlobStore;
import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory;
+import org.apache.flink.runtime.highavailability.nonha.NonHaRegistry;
import org.apache.flink.runtime.jobmanager.StandaloneSubmittedJobGraphStore;
import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
@@ -102,4 +105,14 @@ public class NonHaServices implements HighAvailabilityServices {
public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
return new StandaloneSubmittedJobGraphStore();
}
+
+ @Override
+ public RunningJobsRegistry getRunningJobsRegistry() throws Exception {
+ return new NonHaRegistry();
+ }
+
+ @Override
+ public BlobStore createBlobStore() {
+ return new VoidBlobStore();
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/RunningJobsRegistry.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/RunningJobsRegistry.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/RunningJobsRegistry.java
new file mode 100644
index 0000000..e7c131c
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/RunningJobsRegistry.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.highavailability;
+
+import org.apache.flink.api.common.JobID;
+
+import java.io.IOException;
+
+/**
+ * This registry tracks if a certain job is running.
+ *
+ * <p>This registry is used in highly-available setups with multiple master nodes,
+ * to determine whether a new leader should attempt to recover a certain job (because the
+ * job is still running), or whether the job has already finished successfully (in case of a
+ * finite job) and the leader has only been granted leadership because the previous leader
+ * quit cleanly after the job was finished.
+ */
+public interface RunningJobsRegistry {
+
+ /**
+ * Marks a job as running.
+ *
+ * @param jobID The id of the job.
+ *
+ * @throws IOException Thrown when the communication with the highly-available storage or registry
+ * failed and could not be retried.
+ */
+ void setJobRunning(JobID jobID) throws IOException;
+
+ /**
+ * Marks a job as running.
+ *
+ * @param jobID The id of the job.
+ *
+ * @throws IOException Thrown when the communication with the highly-available storage or registry
+ * failed and could not be retried.
+ */
+ void setJobFinished(JobID jobID) throws IOException;
+
+ /**
+ * Checks whether a job is running.
+ *
+ * @param jobID The id of the job to check.
+ * @return True if the job is still running, false otherwise.
+ *
+ * @throws IOException Thrown when the communication with the highly-available storage or registry
+ * failed and could not be retried.
+ */
+ boolean isJobRunning(JobID jobID) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
index d26b668..3a7736b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
@@ -19,8 +19,15 @@
package org.apache.flink.runtime.highavailability;
import org.apache.curator.framework.CuratorFramework;
+
import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.HighAvailabilityOptions;
+import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.blob.BlobStore;
+import org.apache.flink.runtime.blob.FileSystemBlobStore;
import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
import org.apache.flink.runtime.checkpoint.ZooKeeperCheckpointRecoveryFactory;
import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
@@ -28,12 +35,56 @@ import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.util.ZooKeeperUtils;
+import java.io.IOException;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.StringUtils.isNullOrWhitespaceOnly;
+
/**
- * An implementation of the {@link HighAvailabilityServices} with zookeeper.
+ * An implementation of the {@link HighAvailabilityServices} using Apache ZooKeeper.
+ * The services store data in ZooKeeper's nodes as illustrated by teh following tree structure:
+ *
+ * <pre>
+ * /flink
+ * +/cluster_id_1/resource_manager_lock
+ * | |
+ * | +/job-id-1/job_manager_lock
+ * | | /checkpoints/latest
+ * | | /latest-1
+ * | | /latest-2
+ * | |
+ * | +/job-id-2/job_manager_lock
+ * |
+ * +/cluster_id_2/resource_manager_lock
+ * |
+ * +/job-id-1/job_manager_lock
+ * |/checkpoints/latest
+ * | /latest-1
+ * |/persisted_job_graph
+ * </pre>
+ *
+ * <p>The root path "/flink" is configurable via the option {@link HighAvailabilityOptions#HA_ZOOKEEPER_ROOT}.
+ * This makes sure Flink stores its data under specific subtrees in ZooKeeper, for example to
+ * accommodate specific permission.
+ *
+ * <p>The "cluster_id" part identifies the data stored for a specific Flink "cluster".
+ * This "cluster" can be either a standalone or containerized Flink cluster, or it can be job
+ * on a framework like YARN or Mesos (in a "per-job-cluster" mode).
+ *
+ * <p>In case of a "per-job-cluster" on YARN or Mesos, the cluster-id is generated and configured
+ * automatically by the client or dispatcher that submits the Job to YARN or Mesos.
+ *
+ * <p>In the case of a standalone cluster, that cluster-id needs to be configured via
+ * {@link HighAvailabilityOptions#HA_CLUSTER_ID}. All nodes with the same cluster id will join the same
+ * cluster and participate in the execution of the same set of jobs.
*/
public class ZookeeperHaServices implements HighAvailabilityServices {
- private static final String DEFAULT_RESOURCE_MANAGER_PATH_SUFFIX = "/resource-manager";
+ private static final String RESOURCE_MANAGER_LEADER_PATH = "/resource_manager_lock";
+
+ private static final String JOB_MANAGER_LEADER_PATH = "/job_manager_lock";
+
+ // ------------------------------------------------------------------------
/** The ZooKeeper client to use */
private final CuratorFramework client;
@@ -41,29 +92,37 @@ public class ZookeeperHaServices implements HighAvailabilityServices {
/** The runtime configuration */
private final Configuration configuration;
+ public ZookeeperHaServices(Configuration configuration) {
+ this(ZooKeeperUtils.startCuratorFramework(configuration), configuration);
+ }
+
public ZookeeperHaServices(final CuratorFramework client, final Configuration configuration) {
- this.client = client;
- this.configuration = configuration;
+ this.client = checkNotNull(client);
+ this.configuration = checkNotNull(configuration);
}
+ // ------------------------------------------------------------------------
+ // Services
+ // ------------------------------------------------------------------------
+
@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() throws Exception {
- return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, DEFAULT_RESOURCE_MANAGER_PATH_SUFFIX);
+ return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, RESOURCE_MANAGER_LEADER_PATH);
}
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception {
- return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, getPathSuffixForJob(jobID));
+ return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, getPathForJobManager(jobID));
}
@Override
public LeaderElectionService getResourceManagerLeaderElectionService() throws Exception {
- return ZooKeeperUtils.createLeaderElectionService(client, configuration, DEFAULT_RESOURCE_MANAGER_PATH_SUFFIX);
+ return ZooKeeperUtils.createLeaderElectionService(client, configuration, RESOURCE_MANAGER_LEADER_PATH);
}
@Override
public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) throws Exception {
- return ZooKeeperUtils.createLeaderElectionService(client, configuration, getPathSuffixForJob(jobID));
+ return ZooKeeperUtils.createLeaderElectionService(client, configuration, getPathForJobManager(jobID));
}
@Override
@@ -76,7 +135,43 @@ public class ZookeeperHaServices implements HighAvailabilityServices {
return ZooKeeperUtils.createSubmittedJobGraphs(client, configuration);
}
- private static String getPathSuffixForJob(final JobID jobID) {
- return String.format("/job-managers/%s", jobID);
+ @Override
+ public RunningJobsRegistry getRunningJobsRegistry() throws Exception {
+ throw new UnsupportedOperationException("not yet implemented");
+ }
+
+ @Override
+ public BlobStore createBlobStore() throws IOException {
+ final String storagePath = configuration.getValue(HighAvailabilityOptions.HA_STORAGE_PATH);
+ if (isNullOrWhitespaceOnly(storagePath)) {
+ throw new IllegalConfigurationException("Configuration is missing the mandatory parameter: " +
+ HighAvailabilityOptions.HA_STORAGE_PATH);
+ }
+
+ final Path path;
+ try {
+ path = new Path(storagePath);
+ } catch (Exception e) {
+ throw new IOException("Invalid path for highly available storage (" +
+ HighAvailabilityOptions.HA_STORAGE_PATH.key() + ')', e);
+ }
+
+ final FileSystem fileSystem;
+ try {
+ fileSystem = path.getFileSystem();
+ } catch (Exception e) {
+ throw new IOException("Could not create FileSystem for highly available storage (" +
+ HighAvailabilityOptions.HA_STORAGE_PATH.key() + ')', e);
+ }
+
+ return new FileSystemBlobStore(fileSystem, storagePath);
+ }
+
+ // ------------------------------------------------------------------------
+ // Utilities
+ // ------------------------------------------------------------------------
+
+ private static String getPathForJobManager(final JobID jobID) {
+ return "/" + jobID + JOB_MANAGER_LEADER_PATH;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/NonHaRegistry.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/NonHaRegistry.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/NonHaRegistry.java
new file mode 100644
index 0000000..85dd711
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/NonHaRegistry.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.highavailability.nonha;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.highavailability.RunningJobsRegistry;
+
+import java.util.HashSet;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * A registry for running jobs, not-highly available.
+ */
+public class NonHaRegistry implements RunningJobsRegistry {
+
+ /** The currently running jobs */
+ private final HashSet<JobID> running = new HashSet<>();
+
+ @Override
+ public void setJobRunning(JobID jobID) {
+ checkNotNull(jobID);
+
+ synchronized (running) {
+ running.add(jobID);
+ }
+ }
+
+ @Override
+ public void setJobFinished(JobID jobID) {
+ checkNotNull(jobID);
+
+ synchronized (running) {
+ running.remove(jobID);
+ }
+ }
+
+ @Override
+ public boolean isJobRunning(JobID jobID) {
+ checkNotNull(jobID);
+
+ synchronized (running) {
+ return running.contains(jobID);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java
index 6de4253..25a2a66 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java
@@ -19,9 +19,8 @@
package org.apache.flink.runtime.jobmanager;
import org.apache.flink.api.common.JobExecutionResult;
-import org.apache.flink.runtime.rpc.FatalErrorHandler;
-public interface OnCompletionActions extends FatalErrorHandler {
+public interface OnCompletionActions {
void jobFinished(JobExecutionResult result);
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
index a096932..74ca6f3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
@@ -21,26 +21,38 @@ package org.apache.flink.runtime.jobmaster;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.RunningJobsRegistry;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobmanager.OnCompletionActions;
-import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
-import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
import org.apache.flink.runtime.leaderelection.LeaderContender;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
+import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcService;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.util.UUID;
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* The runner for the job manager. It deals with job level leader election and make underlying job manager
* properly reacted.
*/
-public class JobManagerRunner implements LeaderContender, OnCompletionActions {
+public class JobManagerRunner implements LeaderContender, OnCompletionActions, FatalErrorHandler {
- private final Logger log = LoggerFactory.getLogger(JobManagerRunner.class);
+ private static final Logger log = LoggerFactory.getLogger(JobManagerRunner.class);
+
+ // ------------------------------------------------------------------------
/** Lock to ensure that this runner can deal with leader election event and job completion notifies simultaneously */
private final Object lock = new Object();
@@ -48,52 +60,140 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
/** The job graph needs to run */
private final JobGraph jobGraph;
- private final OnCompletionActions toNotify;
+ /** The listener to notify once the job completes - either successfully or unsuccessfully */
+ private final OnCompletionActions toNotifyOnComplete;
+
+ /** The handler to call in case of fatal (unrecoverable) errors */
+ private final FatalErrorHandler errorHandler;
/** Used to check whether a job needs to be run */
- private final SubmittedJobGraphStore submittedJobGraphStore;
+ private final RunningJobsRegistry runningJobsRegistry;
/** Leader election for this job */
private final LeaderElectionService leaderElectionService;
+ private final JobManagerServices jobManagerServices;
+
private final JobMaster jobManager;
+ private final JobManagerMetricGroup jobManagerMetricGroup;
+
/** flag marking the runner as shut down */
private volatile boolean shutdown;
+ // ------------------------------------------------------------------------
+
public JobManagerRunner(
- final JobGraph jobGraph,
- final Configuration configuration,
- final RpcService rpcService,
- final HighAvailabilityServices haServices,
- final OnCompletionActions toNotify) throws Exception
+ final JobGraph jobGraph,
+ final Configuration configuration,
+ final RpcService rpcService,
+ final HighAvailabilityServices haServices,
+ final OnCompletionActions toNotifyOnComplete,
+ final FatalErrorHandler errorHandler) throws Exception
{
this(jobGraph, configuration, rpcService, haServices,
- JobManagerServices.fromConfiguration(configuration), toNotify);
+ new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(configuration)),
+ toNotifyOnComplete, errorHandler);
}
public JobManagerRunner(
- final JobGraph jobGraph,
- final Configuration configuration,
- final RpcService rpcService,
- final HighAvailabilityServices haServices,
- final JobManagerServices jobManagerServices,
- final OnCompletionActions toNotify) throws Exception
+ final JobGraph jobGraph,
+ final Configuration configuration,
+ final RpcService rpcService,
+ final HighAvailabilityServices haServices,
+ final MetricRegistry metricRegistry,
+ final OnCompletionActions toNotifyOnComplete,
+ final FatalErrorHandler errorHandler) throws Exception
+ {
+ this(jobGraph, configuration, rpcService, haServices,
+ JobManagerServices.fromConfiguration(configuration, haServices),
+ metricRegistry,
+ toNotifyOnComplete, errorHandler);
+ }
+
+ /**
+ *
+ * <p>Exceptions that occur while creating the JobManager or JobManagerRunner are directly
+ * thrown and not reported to the given {@code FatalErrorHandler}.
+ *
+ * <p>This JobManagerRunner assumes that it owns the given {@code JobManagerServices}.
+ * It will shut them down on error and on calls to {@link #shutdown()}.
+ *
+ * @throws Exception Thrown if the runner cannot be set up, because either one of the
+ * required services could not be started, ot the Job could not be initialized.
+ */
+ public JobManagerRunner(
+ final JobGraph jobGraph,
+ final Configuration configuration,
+ final RpcService rpcService,
+ final HighAvailabilityServices haServices,
+ final JobManagerServices jobManagerServices,
+ final MetricRegistry metricRegistry,
+ final OnCompletionActions toNotifyOnComplete,
+ final FatalErrorHandler errorHandler) throws Exception
{
- this.jobGraph = jobGraph;
- this.toNotify = toNotify;
- this.submittedJobGraphStore = haServices.getSubmittedJobGraphStore();
- this.leaderElectionService = haServices.getJobManagerLeaderElectionService(jobGraph.getJobID());
-
- this.jobManager = new JobMaster(
- jobGraph, configuration, rpcService, haServices,
- jobManagerServices.libraryCacheManager,
- jobManagerServices.restartStrategyFactory,
- jobManagerServices.savepointStore,
- jobManagerServices.timeout,
- new Scheduler(jobManagerServices.executorService),
- jobManagerServices.jobManagerMetricGroup,
- this);
+
+ JobManagerMetricGroup jobManagerMetrics = null;
+
+ // make sure we cleanly shut down out JobManager services if initialization fails
+ try {
+ this.jobGraph = checkNotNull(jobGraph);
+ this.toNotifyOnComplete = checkNotNull(toNotifyOnComplete);
+ this.errorHandler = checkNotNull(errorHandler);
+ this.jobManagerServices = checkNotNull(jobManagerServices);
+
+ checkArgument(jobGraph.getNumberOfVertices() > 0, "The given job is empty");
+
+ final String hostAddress = rpcService.getAddress().isEmpty() ? "localhost" : rpcService.getAddress();
+ jobManagerMetrics = new JobManagerMetricGroup(metricRegistry, hostAddress);
+ this.jobManagerMetricGroup = jobManagerMetrics;
+
+ // libraries and class loader first
+ final BlobLibraryCacheManager libraryCacheManager = jobManagerServices.libraryCacheManager;
+ try {
+ libraryCacheManager.registerJob(
+ jobGraph.getJobID(), jobGraph.getUserJarBlobKeys(), jobGraph.getClasspaths());
+ } catch (IOException e) {
+ throw new Exception("Cannot set up the user code libraries: " + e.getMessage(), e);
+ }
+
+ final ClassLoader userCodeLoader = libraryCacheManager.getClassLoader(jobGraph.getJobID());
+ if (userCodeLoader == null) {
+ throw new Exception("The user code class loader could not be initialized.");
+ }
+
+ // high availability services next
+ this.runningJobsRegistry = haServices.getRunningJobsRegistry();
+ this.leaderElectionService = haServices.getJobManagerLeaderElectionService(jobGraph.getJobID());
+
+ // now start the JobManager
+ this.jobManager = new JobMaster(
+ jobGraph, configuration,
+ rpcService,
+ haServices,
+ jobManagerServices.executorService,
+ jobManagerServices.libraryCacheManager,
+ jobManagerServices.restartStrategyFactory,
+ jobManagerServices.rpcAskTimeout,
+ jobManagerMetrics,
+ this,
+ this,
+ userCodeLoader);
+ }
+ catch (Throwable t) {
+ // clean up everything
+ try {
+ jobManagerServices.shutdown();
+ } catch (Throwable tt) {
+ log.error("Error while shutting down JobManager services", tt);
+ }
+
+ if (jobManagerMetrics != null) {
+ jobManagerMetrics.close();
+ }
+
+ throw new JobExecutionException(jobGraph.getJobID(), "Could not set up JobManager", t);
+ }
}
//----------------------------------------------------------------------------------------------
@@ -101,9 +201,6 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
//----------------------------------------------------------------------------------------------
public void start() throws Exception {
- jobManager.init();
- jobManager.start();
-
try {
leaderElectionService.start(this);
}
@@ -114,11 +211,6 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
}
public void shutdown() {
- shutdown(new Exception("The JobManager runner is shutting down"));
- }
-
- public void shutdown(Throwable cause) {
- // TODO what is the cause used for ?
shutdownInternally();
}
@@ -129,12 +221,29 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
if (leaderElectionService != null) {
try {
leaderElectionService.stop();
- } catch (Exception e) {
- log.error("Could not properly shutdown the leader election service.");
+ } catch (Throwable t) {
+ log.error("Could not properly shutdown the leader election service", t);
}
}
- jobManager.shutDown();
+ try {
+ jobManager.shutDown();
+ } catch (Throwable t) {
+ log.error("Error shutting down JobManager", t);
+ }
+
+ try {
+ jobManagerServices.shutdown();
+ } catch (Throwable t) {
+ log.error("Error shutting down JobManager services", t);
+ }
+
+ // make all registered metrics go away
+ try {
+ jobManagerMetricGroup.close();
+ } catch (Throwable t) {
+ log.error("Error while unregistering metrics", t);
+ }
}
}
@@ -148,11 +257,12 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
@Override
public void jobFinished(JobExecutionResult result) {
try {
+ unregisterJobFromHighAvailability();
shutdownInternally();
}
finally {
- if (toNotify != null) {
- toNotify.jobFinished(result);
+ if (toNotifyOnComplete != null) {
+ toNotifyOnComplete.jobFinished(result);
}
}
}
@@ -163,11 +273,12 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
@Override
public void jobFailed(Throwable cause) {
try {
+ unregisterJobFromHighAvailability();
shutdownInternally();
}
finally {
- if (toNotify != null) {
- toNotify.jobFailed(cause);
+ if (toNotifyOnComplete != null) {
+ toNotifyOnComplete.jobFailed(cause);
}
}
}
@@ -178,11 +289,12 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
@Override
public void jobFinishedByOther() {
try {
+ unregisterJobFromHighAvailability();
shutdownInternally();
}
finally {
- if (toNotify != null) {
- toNotify.jobFinishedByOther();
+ if (toNotifyOnComplete != null) {
+ toNotifyOnComplete.jobFinishedByOther();
}
}
}
@@ -192,18 +304,43 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
*/
@Override
public void onFatalError(Throwable exception) {
- // first and in any case, notify our handler, so it can react fast
+ // we log first to make sure an explaining message goes into the log
+ // we even guard the log statement here to increase chances that the error handler
+ // gets the notification on hard critical situations like out-of-memory errors
+ try {
+ log.error("JobManager runner encountered a fatal error.", exception);
+ } catch (Throwable ignored) {}
+
+ // in any case, notify our handler, so it can react fast
try {
- if (toNotify != null) {
- toNotify.onFatalError(exception);
+ if (errorHandler != null) {
+ errorHandler.onFatalError(exception);
}
}
finally {
- log.error("JobManager runner encountered a fatal error.", exception);
+ // the shutdown may not even needed any more, if the fatal error
+ // handler kills the process. that is fine, a process kill cleans up better than anything.
shutdownInternally();
}
}
+ /**
+ * Marks this runner's job as not running. Other JobManager will not recover the job
+ * after this call.
+ *
+ * <p>This method never throws an exception.
+ */
+ private void unregisterJobFromHighAvailability() {
+ try {
+ runningJobsRegistry.setJobFinished(jobGraph.getJobID());
+ }
+ catch (Throwable t) {
+ log.error("Could not un-register from high-availability services job {} ({})." +
+ "Other JobManager's may attempt to recover it and re-execute it.",
+ jobGraph.getName(), jobGraph.getJobID(), t);
+ }
+ }
+
//----------------------------------------------------------------------------------------------
// Leadership methods
//----------------------------------------------------------------------------------------------
@@ -223,15 +360,25 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
// it's okay that job manager wait for the operation complete
leaderElectionService.confirmLeaderSessionID(leaderSessionID);
+ boolean jobRunning;
+ try {
+ jobRunning = runningJobsRegistry.isJobRunning(jobGraph.getJobID());
+ } catch (Throwable t) {
+ log.error("Could not access status (running/finished) of job {}. " +
+ "Falling back to assumption that job is running and attempting recovery...",
+ jobGraph.getJobID(), t);
+ jobRunning = true;
+ }
+
// Double check the leadership after we confirm that, there is a small chance that multiple
// job managers schedule the same job after if they try to recover at the same time.
// This will eventually be noticed, but can not be ruled out from the beginning.
if (leaderElectionService.hasLeadership()) {
- if (isJobFinishedByOthers()) {
+ if (jobRunning) {
+ jobManager.start(leaderSessionID);
+ } else {
log.info("Job {} ({}) already finished by others.", jobGraph.getName(), jobGraph.getJobID());
jobFinishedByOther();
- } else {
- jobManager.getSelf().startJob(leaderSessionID);
}
}
}
@@ -248,7 +395,7 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
log.info("JobManager for job {} ({}) was revoked leadership at {}.",
jobGraph.getName(), jobGraph.getJobID(), getAddress());
- jobManager.getSelf().suspendJob(new Exception("JobManager is no longer the leader."));
+ jobManager.getSelf().suspendExecution(new Exception("JobManager is no longer the leader."));
}
}
@@ -263,11 +410,9 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
onFatalError(exception);
}
- @VisibleForTesting
- boolean isJobFinishedByOthers() {
- // TODO: Fix
- return false;
- }
+ //----------------------------------------------------------------------------------------------
+ // Testing
+ //----------------------------------------------------------------------------------------------
@VisibleForTesting
boolean isShutdown() {
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerServices.java
index e6beba6..fff75d5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerServices.java
@@ -19,13 +19,20 @@
package org.apache.flink.runtime.jobmaster;
import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.checkpoint.savepoint.SavepointStore;
+import org.apache.flink.configuration.IllegalConfigurationException;
+import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.blob.BlobServer;
import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager;
import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory;
-import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.util.ExceptionUtils;
+
+import scala.concurrent.duration.FiniteDuration;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ForkJoinPool;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -40,34 +47,81 @@ public class JobManagerServices {
public final RestartStrategyFactory restartStrategyFactory;
- public final SavepointStore savepointStore;
-
- public final Time timeout;
-
- public final JobManagerMetricGroup jobManagerMetricGroup;
+ public final Time rpcAskTimeout;
public JobManagerServices(
ExecutorService executorService,
BlobLibraryCacheManager libraryCacheManager,
RestartStrategyFactory restartStrategyFactory,
- SavepointStore savepointStore,
- Time timeout,
- JobManagerMetricGroup jobManagerMetricGroup) {
+ Time rpcAskTimeout) {
this.executorService = checkNotNull(executorService);
this.libraryCacheManager = checkNotNull(libraryCacheManager);
this.restartStrategyFactory = checkNotNull(restartStrategyFactory);
- this.savepointStore = checkNotNull(savepointStore);
- this.timeout = checkNotNull(timeout);
- this.jobManagerMetricGroup = checkNotNull(jobManagerMetricGroup);
+ this.rpcAskTimeout = checkNotNull(rpcAskTimeout);
+ }
+
+ /**
+ *
+ * <p>This method makes sure all services are closed or shut down, even when an exception occurred
+ * in the shutdown of one component. The first encountered exception is thrown, with successive
+ * exceptions added as suppressed exceptions.
+ *
+ * @throws Exception The first Exception encountered during shutdown.
+ */
+ public void shutdown() throws Exception {
+ Throwable firstException = null;
+
+ try {
+ executorService.shutdownNow();
+ } catch (Throwable t) {
+ firstException = t;
+ }
+
+ try {
+ libraryCacheManager.shutdown();
+ }
+ catch (Throwable t) {
+ if (firstException == null) {
+ firstException = t;
+ } else {
+ firstException.addSuppressed(t);
+ }
+ }
+
+ if (firstException != null) {
+ ExceptionUtils.rethrowException(firstException, "Error while shutting down JobManager services");
+ }
}
// ------------------------------------------------------------------------
// Creating the components from a configuration
// ------------------------------------------------------------------------
- public static JobManagerServices fromConfiguration(Configuration config) throws Exception {
- // TODO not yet implemented
- return null;
+
+ public static JobManagerServices fromConfiguration(
+ Configuration config,
+ HighAvailabilityServices haServices) throws Exception {
+
+ final BlobServer blobServer = new BlobServer(config, haServices);
+
+ final long cleanupInterval = config.getLong(
+ ConfigConstants.LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL,
+ ConfigConstants.DEFAULT_LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL) * 1000;
+
+ final BlobLibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(blobServer, cleanupInterval);
+
+ final FiniteDuration timeout;
+ try {
+ timeout = AkkaUtils.getTimeout(config);
+ } catch (NumberFormatException e) {
+ throw new IllegalConfigurationException(AkkaUtils.formatDurationParingErrorMessage());
+ }
+
+ return new JobManagerServices(
+ new ForkJoinPool(),
+ libraryCacheManager,
+ RestartStrategyFactory.createRestartStrategyFactory(config),
+ Time.of(timeout.length(), timeout.unit()));
}
}
[45/50] [abbrv] flink git commit: Rebasing fixes on latest master
Posted by tr...@apache.org.
Rebasing fixes on latest master
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5c55266a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5c55266a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5c55266a
Branch: refs/heads/flip-6
Commit: 5c55266a95514a1ac3ff45a8531fd644f3c907e8
Parents: 7ba5b86
Author: Till Rohrmann <tr...@apache.org>
Authored: Thu Oct 20 19:54:12 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:34 2016 +0100
----------------------------------------------------------------------
.../apache/flink/storm/wrappers/BoltWrapperTest.java | 4 ++--
.../checkpoint/CheckpointCoordinatorGateway.java | 5 ++---
.../org/apache/flink/runtime/jobmaster/JobMaster.java | 12 ++++++------
.../taskexecutor/rpc/RpcCheckpointResponder.java | 6 +++---
.../org/apache/flink/runtime/util/ZooKeeperUtils.java | 10 +++-------
.../apache/flink/runtime/jobmanager/JobSubmitTest.java | 4 +++-
.../runtime/operators/testutils/DummyEnvironment.java | 3 ++-
...mulatingAlignedProcessingTimeWindowOperatorTest.java | 3 ++-
8 files changed, 23 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/5c55266a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java
index a7d60a6..7fe9947 100644
--- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java
+++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java
@@ -35,7 +35,7 @@ import org.apache.flink.configuration.UnmodifiableConfiguration;
import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.operators.testutils.UnregisteredTaskMetricsGroup;
-import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import org.apache.flink.storm.util.AbstractTest;
import org.apache.flink.storm.util.SplitStreamType;
import org.apache.flink.storm.util.StormConfig;
@@ -370,7 +370,7 @@ public class BoltWrapperTest extends AbstractTest {
when(env.getTaskInfo()).thenReturn(new TaskInfo("Mock Task", 1, 0, 1, 0));
when(env.getUserClassLoader()).thenReturn(BoltWrapperTest.class.getClassLoader());
when(env.getMetricGroup()).thenReturn(new UnregisteredTaskMetricsGroup());
- when(env.getTaskManagerInfo()).thenReturn(new TaskManagerRuntimeInfo("foo", new Configuration(), "foo"));
+ when(env.getTaskManagerInfo()).thenReturn(new TestingTaskManagerRuntimeInfo());
StreamTask<?, ?> mockTask = mock(StreamTask.class);
when(mockTask.getCheckpointLock()).thenReturn(new Object());
http://git-wip-us.apache.org/repos/asf/flink/blob/5c55266a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java
index 2634006..f9786f6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorGateway.java
@@ -21,15 +21,14 @@ package org.apache.flink.runtime.checkpoint;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.rpc.RpcGateway;
-import org.apache.flink.runtime.state.CheckpointStateHandles;
public interface CheckpointCoordinatorGateway extends RpcGateway {
void acknowledgeCheckpoint(
final JobID jobID,
final ExecutionAttemptID executionAttemptID,
- final CheckpointMetaData checkpointInfo,
- final CheckpointStateHandles checkpointStateHandles);
+ final CheckpointMetaData checkpointMetaData,
+ final SubtaskState subtaskState);
void declineCheckpoint(
final JobID jobID,
http://git-wip-us.apache.org/repos/asf/flink/blob/5c55266a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 56fa3e7..5a7c9a1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -31,6 +31,7 @@ import org.apache.flink.runtime.checkpoint.CheckpointCoordinator;
import org.apache.flink.runtime.checkpoint.CheckpointMetaData;
import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
import org.apache.flink.runtime.checkpoint.CompletedCheckpoint;
+import org.apache.flink.runtime.checkpoint.SubtaskState;
import org.apache.flink.runtime.client.JobExecutionException;
import org.apache.flink.runtime.client.JobSubmissionException;
import org.apache.flink.runtime.client.SerializedJobExecutionResult;
@@ -83,7 +84,6 @@ import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.rpc.StartStoppable;
-import org.apache.flink.runtime.state.CheckpointStateHandles;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
import org.apache.flink.runtime.taskexecutor.slot.SlotOffer;
@@ -529,12 +529,12 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
public void acknowledgeCheckpoint(
final JobID jobID,
final ExecutionAttemptID executionAttemptID,
- final CheckpointMetaData checkpointInfo,
- final CheckpointStateHandles checkpointState)
+ final CheckpointMetaData checkpointMetaData,
+ final SubtaskState subtaskState)
{
final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
final AcknowledgeCheckpoint ackMessage =
- new AcknowledgeCheckpoint(jobID, executionAttemptID, checkpointInfo, checkpointState);
+ new AcknowledgeCheckpoint(jobID, executionAttemptID, checkpointMetaData, subtaskState);
if (checkpointCoordinator != null) {
getRpcService().execute(new Runnable() {
@@ -543,10 +543,10 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
try {
if (!checkpointCoordinator.receiveAcknowledgeMessage(ackMessage)) {
log.info("Received message for non-existing checkpoint {}.",
- checkpointInfo.getCheckpointId());
+ checkpointMetaData.getCheckpointId());
}
} catch (Exception e) {
- log.error("Error in CheckpointCoordinator while processing {}", checkpointInfo, e);
+ log.error("Error in CheckpointCoordinator while processing {}", checkpointMetaData, e);
}
}
});
http://git-wip-us.apache.org/repos/asf/flink/blob/5c55266a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java
index 9669da0..c18da67 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java
@@ -21,8 +21,8 @@ package org.apache.flink.runtime.taskexecutor.rpc;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.checkpoint.CheckpointCoordinatorGateway;
import org.apache.flink.runtime.checkpoint.CheckpointMetaData;
+import org.apache.flink.runtime.checkpoint.SubtaskState;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
-import org.apache.flink.runtime.state.CheckpointStateHandles;
import org.apache.flink.runtime.taskmanager.CheckpointResponder;
import org.apache.flink.util.Preconditions;
@@ -39,13 +39,13 @@ public class RpcCheckpointResponder implements CheckpointResponder {
JobID jobID,
ExecutionAttemptID executionAttemptID,
CheckpointMetaData checkpointMetaData,
- CheckpointStateHandles checkpointStateHandles) {
+ SubtaskState subtaskState) {
checkpointCoordinatorGateway.acknowledgeCheckpoint(
jobID,
executionAttemptID,
checkpointMetaData,
- checkpointStateHandles);
+ subtaskState);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5c55266a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java
index e9777a3..cb5dc31 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java
@@ -161,7 +161,6 @@ public class ZooKeeperUtils {
*/
public static ZooKeeperLeaderRetrievalService createLeaderRetrievalService(
Configuration configuration) {
- {
final CuratorFramework client = startCuratorFramework(configuration);
return createLeaderRetrievalService(client, configuration);
}
@@ -172,11 +171,10 @@ public class ZooKeeperUtils {
* @param client The {@link CuratorFramework} ZooKeeper client to use
* @param configuration {@link Configuration} object containing the configuration values
* @return {@link ZooKeeperLeaderRetrievalService} instance.
- * @throws Exception
*/
public static ZooKeeperLeaderRetrievalService createLeaderRetrievalService(
final CuratorFramework client,
- final Configuration configuration) throws Exception
+ final Configuration configuration)
{
return createLeaderRetrievalService(client, configuration, "");
}
@@ -188,12 +186,11 @@ public class ZooKeeperUtils {
* @param configuration {@link Configuration} object containing the configuration values
* @param pathSuffix The path suffix which we want to append
* @return {@link ZooKeeperLeaderRetrievalService} instance.
- * @throws Exception
*/
public static ZooKeeperLeaderRetrievalService createLeaderRetrievalService(
final CuratorFramework client,
final Configuration configuration,
- final String pathSuffix) throws Exception
+ final String pathSuffix)
{
String leaderPath = ConfigurationUtil.getStringWithDeprecatedKeys(
configuration,
@@ -240,12 +237,11 @@ public class ZooKeeperUtils {
* @param configuration {@link Configuration} object containing the configuration values
* @param pathSuffix The path suffix which we want to append
* @return {@link ZooKeeperLeaderElectionService} instance.
- * @throws Exception
*/
public static ZooKeeperLeaderElectionService createLeaderElectionService(
final CuratorFramework client,
final Configuration configuration,
- final String pathSuffix) throws Exception
+ final String pathSuffix)
{
final String latchPath = ConfigurationUtil.getStringWithDeprecatedKeys(
configuration,
http://git-wip-us.apache.org/repos/asf/flink/blob/5c55266a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
index 260b4d4..3c45ccc 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
@@ -168,6 +168,8 @@ public class JobSubmitTest {
JobVertex jobVertex = new JobVertex("Vertex that fails in initializeOnMaster") {
+ private static final long serialVersionUID = -3540303593784587652L;
+
@Override
public void initializeOnMaster(ClassLoader loader) throws Exception {
throw new RuntimeException("test exception");
@@ -217,7 +219,7 @@ public class JobSubmitTest {
private JobGraph createSimpleJobGraph() {
JobVertex jobVertex = new JobVertex("Vertex");
- jobVertex.setInvokableClass(Tasks.NoOpInvokable.class);
+ jobVertex.setInvokableClass(NoOpInvokable.class);
List<JobVertexID> vertexIdList = Collections.singletonList(jobVertex.getID());
JobGraph jg = new JobGraph("test job", jobVertex);
http://git-wip-us.apache.org/repos/asf/flink/blob/5c55266a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java
index f2616b5..3eba048 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java
@@ -39,6 +39,7 @@ import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.query.KvStateRegistry;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import java.util.Collections;
import java.util.Map;
@@ -92,7 +93,7 @@ public class DummyEnvironment implements Environment {
@Override
public TaskManagerRuntimeInfo getTaskManagerInfo() {
- return new TaskManagerRuntimeInfo("foo", new Configuration(), "foo");
+ return new TestingTaskManagerRuntimeInfo();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/5c55266a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java
index 34f69f8..255a20f 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java
@@ -34,6 +34,7 @@ import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.operators.testutils.UnregisteredTaskMetricsGroup;
import org.apache.flink.runtime.state.StreamStateHandle;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
import org.apache.flink.streaming.api.functions.windowing.WindowFunction;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
@@ -710,7 +711,7 @@ public class AccumulatingAlignedProcessingTimeWindowOperatorTest {
when(env.getTaskInfo()).thenReturn(new TaskInfo("Test task name", 1, 0, 1, 0));
when(env.getUserClassLoader()).thenReturn(AggregatingAlignedProcessingTimeWindowOperatorTest.class.getClassLoader());
when(env.getMetricGroup()).thenReturn(new UnregisteredTaskMetricsGroup());
- when(env.getTaskManagerInfo()).thenReturn(new TaskManagerRuntimeInfo("foo", new Configuration(), "foo"));
+ when(env.getTaskManagerInfo()).thenReturn(new TestingTaskManagerRuntimeInfo());
when(task.getEnvironment()).thenReturn(env);
return task;
[08/50] [abbrv] flink git commit: [FLINK-4606] integrate features of
old ResourceManager
Posted by tr...@apache.org.
[FLINK-4606] integrate features of old ResourceManager
This closes #2540
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5219b407
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5219b407
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5219b407
Branch: refs/heads/flip-6
Commit: 5219b407014a3b3d4cb7b3497229f39df772661e
Parents: ae3a8d5
Author: Maximilian Michels <mx...@apache.org>
Authored: Tue Sep 27 10:38:02 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:28 2016 +0100
----------------------------------------------------------------------
.../InfoMessageListenerRpcGateway.java | 1 -
.../resourcemanager/ResourceManager.java | 146 ++++++++++++-------
.../resourcemanager/ResourceManagerGateway.java | 6 +-
.../ResourceManagerServices.java | 44 ++++++
.../StandaloneResourceManager.java | 19 ++-
.../TaskExecutorRegistration.java | 51 -------
.../registration/TaskExecutorRegistration.java | 51 +++++++
.../slotmanager/SimpleSlotManager.java | 6 -
.../slotmanager/SlotManager.java | 63 ++++++--
.../slotmanager/SlotManagerTest.java | 25 +++-
.../slotmanager/SlotProtocolTest.java | 42 +++---
11 files changed, 295 insertions(+), 159 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/5219b407/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/InfoMessageListenerRpcGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/InfoMessageListenerRpcGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/InfoMessageListenerRpcGateway.java
index c1eeefa..d1373ec 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/InfoMessageListenerRpcGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/InfoMessageListenerRpcGateway.java
@@ -19,7 +19,6 @@
package org.apache.flink.runtime.resourcemanager;
import org.apache.flink.runtime.clusterframework.messages.InfoMessage;
-import org.apache.flink.runtime.jobmaster.JobMaster;
import org.apache.flink.runtime.rpc.RpcGateway;
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/5219b407/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index 83dc4db..190a4de 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -20,14 +20,18 @@ package org.apache.flink.runtime.resourcemanager;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.clusterframework.ApplicationStatus;
import org.apache.flink.runtime.clusterframework.messages.InfoMessage;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.clusterframework.types.ResourceIDRetrievable;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.concurrent.AcceptFunction;
import org.apache.flink.runtime.concurrent.ApplyFunction;
import org.apache.flink.runtime.concurrent.BiFunction;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.instance.InstanceID;
import org.apache.flink.runtime.jobmaster.JobMasterRegistrationSuccess;
import org.apache.flink.runtime.leaderelection.LeaderContender;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
@@ -48,11 +52,10 @@ import org.apache.flink.runtime.util.LeaderRetrievalUtils;
import scala.concurrent.duration.FiniteDuration;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Map;
-import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
+import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -64,36 +67,43 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
* It offers the following methods as part of its rpc interface to interact with the him remotely:
* <ul>
* <li>{@link #registerJobMaster(UUID, UUID, String, JobID)} registers a {@link JobMaster} at the resource manager</li>
- * <li>{@link #requestSlot(SlotRequest)} requests a slot from the resource manager</li>
+ * <li>{@link #requestSlot(UUID, UUID, SlotRequest)} requests a slot from the resource manager</li>
* </ul>
*/
-public abstract class ResourceManager<ResourceManagerGateway, WorkerType extends TaskExecutorRegistration> extends RpcEndpoint implements LeaderContender {
+public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
+ extends RpcEndpoint<ResourceManagerGateway>
+ implements LeaderContender {
/** The exit code with which the process is stopped in case of a fatal error */
protected static final int EXIT_CODE_FATAL_ERROR = -13;
private final Map<JobID, JobMasterGateway> jobMasterGateways;
- private final Set<LeaderRetrievalListener> jobMasterLeaderRetrievalListeners;
+ private final Map<JobID, JobMasterLeaderListener> jobMasterLeaderRetrievalListeners;
private final Map<ResourceID, WorkerType> taskExecutorGateways;
private final HighAvailabilityServices highAvailabilityServices;
- private LeaderElectionService leaderElectionService;
-
private final SlotManager slotManager;
+ private LeaderElectionService leaderElectionService;
+
private UUID leaderSessionID;
private Map<String, InfoMessageListenerRpcGateway> infoMessageListeners;
- public ResourceManager(RpcService rpcService, HighAvailabilityServices highAvailabilityServices, SlotManager slotManager) {
+ private final Time timeout = Time.seconds(5);
+
+ public ResourceManager(
+ RpcService rpcService,
+ HighAvailabilityServices highAvailabilityServices,
+ SlotManager slotManager) {
super(rpcService);
this.highAvailabilityServices = checkNotNull(highAvailabilityServices);
this.jobMasterGateways = new HashMap<>();
this.slotManager = checkNotNull(slotManager);
- this.jobMasterLeaderRetrievalListeners = new HashSet<>();
+ this.jobMasterLeaderRetrievalListeners = new HashMap<>();
this.taskExecutorGateways = new HashMap<>();
infoMessageListeners = new HashMap<>();
}
@@ -105,6 +115,7 @@ public abstract class ResourceManager<ResourceManagerGateway, WorkerType extends
super.start();
leaderElectionService = highAvailabilityServices.getResourceManagerLeaderElectionService();
leaderElectionService.start(this);
+ slotManager.setupResourceManagerServices(new DefaultResourceManagerServices());
// framework specific initialization
initialize();
} catch (Throwable e) {
@@ -117,7 +128,7 @@ public abstract class ResourceManager<ResourceManagerGateway, WorkerType extends
public void shutDown() {
try {
leaderElectionService.stop();
- for(JobID jobID : jobMasterGateways.keySet()) {
+ for (JobID jobID : jobMasterGateways.keySet()) {
highAvailabilityServices.getJobMasterLeaderRetriever(jobID).stop();
}
super.shutDown();
@@ -189,15 +200,17 @@ public abstract class ResourceManager<ResourceManagerGateway, WorkerType extends
if (throwable != null) {
return new RegistrationResponse.Decline(throwable.getMessage());
} else {
- JobMasterLeaderListener jobMasterLeaderListener = new JobMasterLeaderListener(jobID);
- try {
- LeaderRetrievalService jobMasterLeaderRetriever = highAvailabilityServices.getJobMasterLeaderRetriever(jobID);
- jobMasterLeaderRetriever.start(jobMasterLeaderListener);
- } catch (Exception e) {
- log.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
- return new RegistrationResponse.Decline("Failed to retrieve JobMasterLeaderRetriever");
+ if (!jobMasterLeaderRetrievalListeners.containsKey(jobID)) {
+ JobMasterLeaderListener jobMasterLeaderListener = new JobMasterLeaderListener(jobID);
+ try {
+ LeaderRetrievalService jobMasterLeaderRetriever = highAvailabilityServices.getJobMasterLeaderRetriever(jobID);
+ jobMasterLeaderRetriever.start(jobMasterLeaderListener);
+ } catch (Exception e) {
+ log.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
+ return new RegistrationResponse.Decline("Failed to retrieve JobMasterLeaderRetriever");
+ }
+ jobMasterLeaderRetrievalListeners.put(jobID, jobMasterLeaderListener);
}
- jobMasterLeaderRetrievalListeners.add(jobMasterLeaderListener);
final JobMasterGateway existingGateway = jobMasterGateways.put(jobID, jobMasterGateway);
if (existingGateway != null) {
log.info("Replacing gateway for registered JobID {}.", jobID);
@@ -232,7 +245,6 @@ public abstract class ResourceManager<ResourceManagerGateway, WorkerType extends
resourceID, taskExecutorAddress, leaderSessionID, resourceManagerLeaderId);
throw new Exception("Invalid leader session id");
}
-
return getRpcService().connect(taskExecutorAddress, TaskExecutorGateway.class).get(5, TimeUnit.SECONDS);
}
}).handleAsync(new BiFunction<TaskExecutorGateway, Throwable, RegistrationResponse>() {
@@ -241,24 +253,14 @@ public abstract class ResourceManager<ResourceManagerGateway, WorkerType extends
if (throwable != null) {
return new RegistrationResponse.Decline(throwable.getMessage());
} else {
- WorkerType startedWorker = taskExecutorGateways.get(resourceID);
- if(startedWorker != null) {
- String oldWorkerAddress = startedWorker.getTaskExecutorGateway().getAddress();
- if (taskExecutorAddress.equals(oldWorkerAddress)) {
- log.warn("Receive a duplicate registration from TaskExecutor {} at ({})", resourceID, taskExecutorAddress);
- } else {
- log.warn("Receive a duplicate registration from TaskExecutor {} at different address, previous ({}), new ({})",
- resourceID, oldWorkerAddress, taskExecutorAddress);
- // TODO :: suggest old taskExecutor to stop itself
- slotManager.notifyTaskManagerFailure(resourceID);
- startedWorker = workerStarted(resourceID, taskExecutorGateway);
- taskExecutorGateways.put(resourceID, startedWorker);
- }
- } else {
- startedWorker = workerStarted(resourceID, taskExecutorGateway);
- taskExecutorGateways.put(resourceID, startedWorker);
+ WorkerType oldWorker = taskExecutorGateways.remove(resourceID);
+ if (oldWorker != null) {
+ // TODO :: suggest old taskExecutor to stop itself
+ slotManager.notifyTaskManagerFailure(resourceID);
}
- return new TaskExecutorRegistrationSuccess(startedWorker.getInstanceID(), 5000);
+ WorkerType newWorker = workerStarted(resourceID);
+ taskExecutorGateways.put(resourceID, newWorker);
+ return new TaskExecutorRegistrationSuccess(new InstanceID(), 5000);
}
}
}, getMainThreadExecutor());
@@ -271,11 +273,20 @@ public abstract class ResourceManager<ResourceManagerGateway, WorkerType extends
* @return Slot assignment
*/
@RpcMethod
- public SlotRequestReply requestSlot(SlotRequest slotRequest) {
- final JobID jobId = slotRequest.getJobId();
- final JobMasterGateway jobMasterGateway = jobMasterGateways.get(jobId);
+ public SlotRequestReply requestSlot(
+ UUID jobMasterLeaderID,
+ UUID resourceManagerLeaderID,
+ SlotRequest slotRequest) {
+
+ JobID jobId = slotRequest.getJobId();
+ JobMasterGateway jobMasterGateway = jobMasterGateways.get(jobId);
+ JobMasterLeaderListener jobMasterLeaderListener = jobMasterLeaderRetrievalListeners.get(jobId);
+
+ UUID leaderID = jobMasterLeaderListener.getLeaderID();
- if (jobMasterGateway != null) {
+ if (jobMasterGateway != null
+ && jobMasterLeaderID.equals(leaderID)
+ && resourceManagerLeaderID.equals(leaderSessionID)) {
return slotManager.requestSlot(slotRequest);
} else {
log.info("Ignoring slot request for unknown JobMaster with JobID {}", jobId);
@@ -379,7 +390,7 @@ public abstract class ResourceManager<ResourceManagerGateway, WorkerType extends
}
/**
- * Shutdowns cluster
+ * Cleanup application and shut down cluster
*
* @param finalStatus
* @param optionalDiagnostics
@@ -446,17 +457,11 @@ public abstract class ResourceManager<ResourceManagerGateway, WorkerType extends
protected abstract void initialize() throws Exception;
/**
- * Callback when a task executor register.
+ * Notifies the resource master of a fatal error.
*
- * @param resourceID The worker resource id
- * @param taskExecutorGateway the task executor gateway
- */
- protected abstract WorkerType workerStarted(ResourceID resourceID, TaskExecutorGateway taskExecutorGateway);
-
- /**
- * Callback when a resource manager faced a fatal error
- * @param message
- * @param error
+ * <p><b>IMPORTANT:</b> This should not cleanly shut down this master, but exit it in
+ * such a way that a high-availability setting would restart this or fail over
+ * to another master.
*/
protected abstract void fatalError(String message, Throwable error);
@@ -472,6 +477,19 @@ public abstract class ResourceManager<ResourceManagerGateway, WorkerType extends
*/
protected abstract void shutDownApplication(ApplicationStatus finalStatus, String optionalDiagnostics);
+ /**
+ * Allocates a resource using the resource profile.
+ * @param resourceProfile The resource description
+ */
+ @VisibleForTesting
+ public abstract void startNewWorker(ResourceProfile resourceProfile);
+
+ /**
+ * Callback when a worker was started.
+ * @param resourceID The worker resource id
+ */
+ protected abstract WorkerType workerStarted(ResourceID resourceID);
+
// ------------------------------------------------------------------------
// Info messaging
// ------------------------------------------------------------------------
@@ -489,6 +507,24 @@ public abstract class ResourceManager<ResourceManagerGateway, WorkerType extends
});
}
+ private class DefaultResourceManagerServices implements ResourceManagerServices {
+
+ @Override
+ public void allocateResource(ResourceProfile resourceProfile) {
+ ResourceManager.this.startNewWorker(resourceProfile);
+ }
+
+ @Override
+ public Executor getAsyncExecutor() {
+ return ResourceManager.this.getRpcService().getExecutor();
+ }
+
+ @Override
+ public Executor getExecutor() {
+ return ResourceManager.this.getMainThreadExecutor();
+ }
+ }
+
private static class JobMasterLeaderListener implements LeaderRetrievalListener {
private final JobID jobID;
@@ -498,6 +534,14 @@ public abstract class ResourceManager<ResourceManagerGateway, WorkerType extends
this.jobID = jobID;
}
+ public JobID getJobID() {
+ return jobID;
+ }
+
+ public UUID getLeaderID() {
+ return leaderID;
+ }
+
@Override
public void notifyLeaderAddress(final String leaderAddress, final UUID leaderSessionID) {
this.leaderID = leaderSessionID;
http://git-wip-us.apache.org/repos/asf/flink/blob/5219b407/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
index 7c44006..87303a1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
@@ -59,7 +59,11 @@ public interface ResourceManagerGateway extends RpcGateway {
* @param slotRequest Slot request
* @return Future slot assignment
*/
- Future<SlotRequestReply> requestSlot(SlotRequest slotRequest);
+ Future<SlotRequestReply> requestSlot(
+ UUID jobMasterLeaderID,
+ UUID resourceManagerLeaderID,
+ SlotRequest slotRequest,
+ @RpcTimeout Time timeout);
/**
* Register a {@link org.apache.flink.runtime.taskexecutor.TaskExecutor} at the resource manager.
http://git-wip-us.apache.org/repos/asf/flink/blob/5219b407/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
new file mode 100644
index 0000000..30994dc
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.runtime.resourcemanager;
+
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+
+import java.util.concurrent.Executor;
+
+/**
+ * Interface which provides access to services of the ResourceManager.
+ */
+public interface ResourceManagerServices {
+
+ /**
+ * Allocates a resource according to the resource profile.
+ */
+ void allocateResource(ResourceProfile resourceProfile);
+
+ /**
+ * Gets the async excutor which executes outside of the main thread of the ResourceManager
+ */
+ Executor getAsyncExecutor();
+
+ /**
+ * Gets the executor which executes in the main thread of the ResourceManager
+ */
+ Executor getExecutor();
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5219b407/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
index 84db1ee..deca8d3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
@@ -20,17 +20,18 @@ package org.apache.flink.runtime.resourcemanager;
import org.apache.flink.runtime.clusterframework.ApplicationStatus;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.instance.InstanceID;
import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
/**
* A standalone implementation of the resource manager. Used when the system is started in
* standalone mode (via scripts), rather than via a resource framework like YARN or Mesos.
+ *
+ * This ResourceManager doesn't acquire new resources.
*/
-public class StandaloneResourceManager extends ResourceManager<ResourceManagerGateway, TaskExecutorRegistration> {
+public class StandaloneResourceManager extends ResourceManager<ResourceID> {
public StandaloneResourceManager(RpcService rpcService,
HighAvailabilityServices highAvailabilityServices,
@@ -51,14 +52,16 @@ public class StandaloneResourceManager extends ResourceManager<ResourceManagerGa
}
@Override
- protected TaskExecutorRegistration workerStarted(ResourceID resourceID, TaskExecutorGateway taskExecutorGateway) {
- InstanceID instanceID = new InstanceID();
- TaskExecutorRegistration taskExecutorRegistration = new TaskExecutorRegistration(taskExecutorGateway, instanceID);
- return taskExecutorRegistration;
+ protected void shutDownApplication(ApplicationStatus finalStatus, String optionalDiagnostics) {
}
@Override
- protected void shutDownApplication(ApplicationStatus finalStatus, String optionalDiagnostics) {
+ public void startNewWorker(ResourceProfile resourceProfile) {
+ }
+ @Override
+ protected ResourceID workerStarted(ResourceID resourceID) {
+ return resourceID;
}
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5219b407/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/TaskExecutorRegistration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/TaskExecutorRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/TaskExecutorRegistration.java
deleted file mode 100644
index f8dfdc7..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/TaskExecutorRegistration.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.resourcemanager;
-
-import org.apache.flink.runtime.instance.InstanceID;
-import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
-
-import java.io.Serializable;
-
-/**
- * This class is responsible for group the TaskExecutorGateway and the InstanceID of a registered task executor.
- */
-public class TaskExecutorRegistration implements Serializable {
-
- private static final long serialVersionUID = -2062957799469434614L;
-
- private TaskExecutorGateway taskExecutorGateway;
-
- private InstanceID instanceID;
-
- public TaskExecutorRegistration(TaskExecutorGateway taskExecutorGateway,
- InstanceID instanceID) {
- this.taskExecutorGateway = taskExecutorGateway;
- this.instanceID = instanceID;
- }
-
- public InstanceID getInstanceID() {
- return instanceID;
- }
-
- public TaskExecutorGateway getTaskExecutorGateway() {
- return taskExecutorGateway;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/5219b407/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorRegistration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorRegistration.java
new file mode 100644
index 0000000..6b21f5c
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/TaskExecutorRegistration.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.registration;
+
+import org.apache.flink.runtime.instance.InstanceID;
+import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
+
+import java.io.Serializable;
+
+/**
+ * This class is responsible for group the TaskExecutorGateway and the InstanceID of a registered task executor.
+ */
+public class TaskExecutorRegistration implements Serializable {
+
+ private static final long serialVersionUID = -2062957799469434614L;
+
+ private TaskExecutorGateway taskExecutorGateway;
+
+ private InstanceID instanceID;
+
+ public TaskExecutorRegistration(TaskExecutorGateway taskExecutorGateway,
+ InstanceID instanceID) {
+ this.taskExecutorGateway = taskExecutorGateway;
+ this.instanceID = instanceID;
+ }
+
+ public InstanceID getInstanceID() {
+ return instanceID;
+ }
+
+ public TaskExecutorGateway getTaskExecutorGateway() {
+ return taskExecutorGateway;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5219b407/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SimpleSlotManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SimpleSlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SimpleSlotManager.java
index ef5ce31..ae1de5a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SimpleSlotManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SimpleSlotManager.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.resourcemanager.slotmanager;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
-import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.clusterframework.types.ResourceSlot;
import org.apache.flink.runtime.clusterframework.types.SlotID;
import org.apache.flink.runtime.resourcemanager.SlotRequest;
@@ -51,9 +50,4 @@ public class SimpleSlotManager extends SlotManager {
}
}
- @Override
- protected void allocateContainer(ResourceProfile resourceProfile) {
- // TODO
- }
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5219b407/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
index a6d2196..a56b2f6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
@@ -22,16 +22,18 @@ import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.clusterframework.types.ResourceSlot;
import org.apache.flink.runtime.clusterframework.types.SlotID;
+import org.apache.flink.runtime.concurrent.BiFunction;
import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerServices;
import org.apache.flink.runtime.resourcemanager.SlotRequest;
import org.apache.flink.runtime.resourcemanager.SlotRequestRegistered;
import org.apache.flink.runtime.resourcemanager.SlotRequestReply;
import org.apache.flink.runtime.taskexecutor.SlotReport;
import org.apache.flink.runtime.taskexecutor.SlotStatus;
import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
+import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -82,6 +84,9 @@ public abstract class SlotManager {
/** The current leader id set by the ResourceManager */
private UUID leaderID;
+ /** The Resource allocation provider */
+ private ResourceManagerServices resourceManagerServices;
+
public SlotManager() {
this.registeredSlots = new HashMap<>(16);
this.pendingSlotRequests = new LinkedHashMap<>(16);
@@ -91,6 +96,16 @@ public abstract class SlotManager {
this.timeout = Time.seconds(10);
}
+ /**
+ * Initializes the resource supplier which is needed to request new resources.
+ */
+ public void setupResourceManagerServices(ResourceManagerServices resourceManagerServices) {
+ if (this.resourceManagerServices != null) {
+ throw new IllegalStateException("ResourceManagerServices may only be set once.");
+ }
+ this.resourceManagerServices = resourceManagerServices;
+ }
+
// ------------------------------------------------------------------------
// slot managements
@@ -120,17 +135,32 @@ public abstract class SlotManager {
// record this allocation in bookkeeping
allocationMap.addAllocation(slot.getSlotId(), allocationId);
-
// remove selected slot from free pool
- freeSlots.remove(slot.getSlotId());
+ final ResourceSlot removedSlot = freeSlots.remove(slot.getSlotId());
final Future<SlotRequestReply> slotRequestReplyFuture =
slot.getTaskExecutorGateway().requestSlot(allocationId, leaderID, timeout);
- // TODO handle timeouts and response
+
+ slotRequestReplyFuture.handleAsync(new BiFunction<SlotRequestReply, Throwable, Object>() {
+ @Override
+ public Object apply(SlotRequestReply slotRequestReply, Throwable throwable) {
+ if (throwable != null) {
+ // we failed, put the slot and the request back again
+ if (allocationMap.isAllocated(slot.getSlotId())) {
+ // only re-add if the slot hasn't been removed in the meantime
+ freeSlots.put(slot.getSlotId(), removedSlot);
+ }
+ pendingSlotRequests.put(allocationId, request);
+ }
+ return null;
+ }
+ }, resourceManagerServices.getExecutor());
} else {
LOG.info("Cannot fulfil slot request, try to allocate a new container for it, " +
"AllocationID:{}, JobID:{}", allocationId, request.getJobId());
- allocateContainer(request.getResourceProfile());
+ Preconditions.checkState(resourceManagerServices != null,
+ "Attempted to allocate resources but no ResourceManagerServices set.");
+ resourceManagerServices.allocateResource(request.getResourceProfile());
pendingSlotRequests.put(allocationId, request);
}
@@ -343,7 +373,7 @@ public abstract class SlotManager {
if (chosenRequest != null) {
final AllocationID allocationId = chosenRequest.getAllocationId();
- pendingSlotRequests.remove(allocationId);
+ final SlotRequest removedSlotRequest = pendingSlotRequests.remove(allocationId);
LOG.info("Assigning SlotID({}) to AllocationID({}), JobID:{}", freeSlot.getSlotId(),
allocationId, chosenRequest.getJobId());
@@ -351,7 +381,19 @@ public abstract class SlotManager {
final Future<SlotRequestReply> slotRequestReplyFuture =
freeSlot.getTaskExecutorGateway().requestSlot(allocationId, leaderID, timeout);
- // TODO handle timeouts and response
+
+ slotRequestReplyFuture.handleAsync(new BiFunction<SlotRequestReply, Throwable, Object>() {
+ @Override
+ public Object apply(SlotRequestReply slotRequestReply, Throwable throwable) {
+ if (throwable != null) {
+ // we failed, add the request back again
+ if (allocationMap.isAllocated(freeSlot.getSlotId())) {
+ pendingSlotRequests.put(allocationId, removedSlotRequest);
+ }
+ }
+ return null;
+ }
+ }, resourceManagerServices.getExecutor());
} else {
freeSlots.put(freeSlot.getSlotId(), freeSlot);
}
@@ -417,13 +459,6 @@ public abstract class SlotManager {
protected abstract SlotRequest chooseRequestToFulfill(final ResourceSlot offeredSlot,
final Map<AllocationID, SlotRequest> pendingRequests);
- /**
- * The framework specific code for allocating a container for specified resource profile.
- *
- * @param resourceProfile The resource profile
- */
- protected abstract void allocateContainer(final ResourceProfile resourceProfile);
-
// ------------------------------------------------------------------------
// Helper classes
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/5219b407/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
index 9ee9690..0fed79e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
@@ -19,12 +19,16 @@
package org.apache.flink.runtime.resourcemanager.slotmanager;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.clusterframework.types.ResourceSlot;
import org.apache.flink.runtime.clusterframework.types.SlotID;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerServices;
import org.apache.flink.runtime.resourcemanager.SlotRequest;
+import org.apache.flink.runtime.resourcemanager.SlotRequestReply;
import org.apache.flink.runtime.taskexecutor.SlotStatus;
import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
import org.junit.BeforeClass;
@@ -34,10 +38,13 @@ import org.mockito.Mockito;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Executor;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
public class SlotManagerTest {
@@ -57,6 +64,8 @@ public class SlotManagerTest {
@BeforeClass
public static void setUp() {
taskExecutorGateway = Mockito.mock(TaskExecutorGateway.class);
+ Mockito.when(taskExecutorGateway.requestSlot(any(AllocationID.class), any(UUID.class), any(Time.class)))
+ .thenReturn(new FlinkCompletableFuture<SlotRequestReply>());
}
/**
@@ -498,12 +507,13 @@ public class SlotManagerTest {
// testing classes
// ------------------------------------------------------------------------
- private static class TestingSlotManager extends SlotManager {
+ private static class TestingSlotManager extends SlotManager implements ResourceManagerServices {
private final List<ResourceProfile> allocatedContainers;
TestingSlotManager() {
this.allocatedContainers = new LinkedList<>();
+ setupResourceManagerServices(this);
}
/**
@@ -543,12 +553,23 @@ public class SlotManagerTest {
}
@Override
- protected void allocateContainer(ResourceProfile resourceProfile) {
+ public void allocateResource(ResourceProfile resourceProfile) {
allocatedContainers.add(resourceProfile);
}
+ @Override
+ public Executor getAsyncExecutor() {
+ return Mockito.mock(Executor.class);
+ }
+
+ @Override
+ public Executor getExecutor() {
+ return Mockito.mock(Executor.class);
+ }
+
List<ResourceProfile> getAllocatedContainers() {
return allocatedContainers;
}
+
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5219b407/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
index ff25897..e3018c9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
@@ -24,18 +24,14 @@ import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.clusterframework.types.SlotID;
import org.apache.flink.runtime.concurrent.Future;
-import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.highavailability.NonHaServices;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.concurrent.impl.FlinkFuture;
import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
import org.apache.flink.runtime.jobmaster.JobMasterGateway;
-import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
import org.apache.flink.runtime.registration.RegistrationResponse;
-import org.apache.flink.runtime.resourcemanager.ResourceManager;
-import org.apache.flink.runtime.resourcemanager.SlotRequest;
-import org.apache.flink.runtime.resourcemanager.SlotRequestReply;
-import org.apache.flink.runtime.resourcemanager.StandaloneResourceManager;
+import org.apache.flink.runtime.resourcemanager.*;
import org.apache.flink.runtime.rpc.TestingSerialRpcService;
import org.apache.flink.runtime.taskexecutor.SlotReport;
import org.apache.flink.runtime.taskexecutor.SlotStatus;
@@ -47,9 +43,12 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
import java.util.Collections;
import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import static org.mockito.Matchers.any;
@@ -99,9 +98,9 @@ public class SlotProtocolTest extends TestLogger {
TestingLeaderElectionService rmLeaderElectionService =
configureHA(testingHaServices, jobID, rmAddress, rmLeaderID, jmAddress, jmLeaderID);
- TestingSlotManager slotManager = Mockito.spy(new TestingSlotManager());
+ SlotManager slotManager = Mockito.spy(new SimpleSlotManager());
ResourceManager resourceManager =
- new StandaloneResourceManager(testRpcService, testingHaServices, slotManager);
+ Mockito.spy(new StandaloneResourceManager(testRpcService, testingHaServices, slotManager));
resourceManager.start();
rmLeaderElectionService.isLeader(rmLeaderID);
@@ -118,7 +117,7 @@ public class SlotProtocolTest extends TestLogger {
SlotRequest slotRequest = new SlotRequest(jobID, allocationID, resourceProfile);
SlotRequestReply slotRequestReply =
- resourceManager.requestSlot(slotRequest);
+ resourceManager.requestSlot(jmLeaderID, rmLeaderID, slotRequest);
// 1) SlotRequest is routed to the SlotManager
verify(slotManager).requestSlot(slotRequest);
@@ -129,13 +128,15 @@ public class SlotProtocolTest extends TestLogger {
allocationID);
// 3) SlotRequest leads to a container allocation
- verify(slotManager, timeout(5000)).allocateContainer(resourceProfile);
+ verify(resourceManager, timeout(5000)).startNewWorker(resourceProfile);
Assert.assertFalse(slotManager.isAllocated(allocationID));
// slot becomes available
final String tmAddress = "/tm1";
TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+ Mockito.when(taskExecutorGateway.requestSlot(any(AllocationID.class), any(UUID.class), any(Time.class)))
+ .thenReturn(new FlinkCompletableFuture<SlotRequestReply>());
testRpcService.registerGateway(tmAddress, taskExecutorGateway);
final ResourceID resourceID = ResourceID.generate();
@@ -176,11 +177,13 @@ public class SlotProtocolTest extends TestLogger {
configureHA(testingHaServices, jobID, rmAddress, rmLeaderID, jmAddress, jmLeaderID);
TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
+ Mockito.when(taskExecutorGateway.requestSlot(any(AllocationID.class), any(UUID.class), any(Time.class)))
+ .thenReturn(new FlinkCompletableFuture<SlotRequestReply>());
testRpcService.registerGateway(tmAddress, taskExecutorGateway);
- TestingSlotManager slotManager = Mockito.spy(new TestingSlotManager());
+ SlotManager slotManager = Mockito.spy(new SimpleSlotManager());
ResourceManager resourceManager =
- new StandaloneResourceManager(testRpcService, testingHaServices, slotManager);
+ Mockito.spy(new StandaloneResourceManager(testRpcService, testingHaServices, slotManager));
resourceManager.start();
rmLeaderElectionService.isLeader(rmLeaderID);
@@ -207,7 +210,7 @@ public class SlotProtocolTest extends TestLogger {
SlotRequest slotRequest = new SlotRequest(jobID, allocationID, resourceProfile);
SlotRequestReply slotRequestReply =
- resourceManager.requestSlot(slotRequest);
+ resourceManager.requestSlot(jmLeaderID, rmLeaderID, slotRequest);
// 1) a SlotRequest is routed to the SlotManager
verify(slotManager).requestSlot(slotRequest);
@@ -241,15 +244,4 @@ public class SlotProtocolTest extends TestLogger {
return rmLeaderElectionService;
}
- private static class TestingSlotManager extends SimpleSlotManager {
-
- // change visibility of function to public for testing
- @Override
- public void allocateContainer(ResourceProfile resourceProfile) {
- super.allocateContainer(resourceProfile);
- }
-
-
- }
-
}
[14/50] [abbrv] flink git commit: [FLINK-4657] Implement
HighAvailabilityServices based on ZooKeeper
Posted by tr...@apache.org.
[FLINK-4657] Implement HighAvailabilityServices based on ZooKeeper
[FLINK-4657] Implement a few rpc calls for JobMaster
[FLINK-4657][cluster management] Address review comments
[FLINK-4657][cluster management] Throw exception when error occurred when request input split
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c7e1d8f6
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c7e1d8f6
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c7e1d8f6
Branch: refs/heads/flip-6
Commit: c7e1d8f62802276313d51ce28ab562e10215ed4e
Parents: dd8c9b9
Author: Kurt Young <yk...@gmail.com>
Authored: Mon Sep 26 10:59:16 2016 +0800
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:29 2016 +0100
----------------------------------------------------------------------
.../HighAvailabilityServices.java | 7 +-
.../runtime/highavailability/NonHaServices.java | 4 +-
.../highavailability/ZookeeperHaServices.java | 82 ++++++++++
.../StandaloneSubmittedJobGraphStore.java | 5 +
.../jobmanager/SubmittedJobGraphStore.java | 8 +
.../ZooKeeperSubmittedJobGraphStore.java | 7 +
.../runtime/jobmaster/JobManagerRunner.java | 18 +--
.../flink/runtime/jobmaster/JobMaster.java | 161 ++++++++++++++++++-
.../runtime/jobmaster/JobMasterGateway.java | 54 ++++++-
.../jobmaster/message/NextInputSplit.java | 39 +++++
.../resourcemanager/ResourceManager.java | 6 +-
.../flink/runtime/util/ZooKeeperUtils.java | 80 +++++++--
.../flink/runtime/taskmanager/TaskManager.scala | 6 +-
.../TestingHighAvailabilityServices.java | 20 +--
.../jobmanager/JobManagerHARecoveryTest.java | 3 +-
.../jobmaster/JobManagerRunnerMockTest.java | 11 +-
.../slotmanager/SlotProtocolTest.java | 2 +-
17 files changed, 454 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
index d67e927..a26886a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
@@ -32,6 +32,7 @@ import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
* <li>JobManager leader election and leader retrieval</li>
* <li>Persistence for checkpoint metadata</li>
* <li>Registering the latest completed checkpoint(s)</li>
+ * <li>Persistence for submitted job graph</li>
* </ul>
*/
public interface HighAvailabilityServices {
@@ -48,12 +49,10 @@ public interface HighAvailabilityServices {
* @return
* @throws Exception
*/
- LeaderRetrievalService getJobMasterLeaderRetriever(JobID jobID) throws Exception;
+ LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception;
/**
* Gets the leader election service for the cluster's resource manager.
- * @return
- * @throws Exception
*/
LeaderElectionService getResourceManagerLeaderElectionService() throws Exception;
@@ -62,7 +61,7 @@ public interface HighAvailabilityServices {
*
* @param jobID The identifier of the job running the election.
*/
- LeaderElectionService getJobMasterLeaderElectionService(JobID jobID) throws Exception;
+ LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) throws Exception;
/**
* Gets the checkpoint recovery factory for the job manager
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
index a2c9cc4..2c6295c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
@@ -79,7 +79,7 @@ public class NonHaServices implements HighAvailabilityServices {
}
@Override
- public LeaderRetrievalService getJobMasterLeaderRetriever(JobID jobID) throws Exception {
+ public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception {
return new StandaloneLeaderRetrievalService(jobMastersAddress.get(jobID), new UUID(0, 0));
}
@@ -89,7 +89,7 @@ public class NonHaServices implements HighAvailabilityServices {
}
@Override
- public LeaderElectionService getJobMasterLeaderElectionService(JobID jobID) throws Exception {
+ public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) throws Exception {
return new StandaloneLeaderElectionService();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
new file mode 100644
index 0000000..d26b668
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.highavailability;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
+import org.apache.flink.runtime.checkpoint.ZooKeeperCheckpointRecoveryFactory;
+import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
+import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.runtime.util.ZooKeeperUtils;
+
+/**
+ * An implementation of the {@link HighAvailabilityServices} with zookeeper.
+ */
+public class ZookeeperHaServices implements HighAvailabilityServices {
+
+ private static final String DEFAULT_RESOURCE_MANAGER_PATH_SUFFIX = "/resource-manager";
+
+ /** The ZooKeeper client to use */
+ private final CuratorFramework client;
+
+ /** The runtime configuration */
+ private final Configuration configuration;
+
+ public ZookeeperHaServices(final CuratorFramework client, final Configuration configuration) {
+ this.client = client;
+ this.configuration = configuration;
+ }
+
+ @Override
+ public LeaderRetrievalService getResourceManagerLeaderRetriever() throws Exception {
+ return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, DEFAULT_RESOURCE_MANAGER_PATH_SUFFIX);
+ }
+
+ @Override
+ public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception {
+ return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, getPathSuffixForJob(jobID));
+ }
+
+ @Override
+ public LeaderElectionService getResourceManagerLeaderElectionService() throws Exception {
+ return ZooKeeperUtils.createLeaderElectionService(client, configuration, DEFAULT_RESOURCE_MANAGER_PATH_SUFFIX);
+ }
+
+ @Override
+ public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) throws Exception {
+ return ZooKeeperUtils.createLeaderElectionService(client, configuration, getPathSuffixForJob(jobID));
+ }
+
+ @Override
+ public CheckpointRecoveryFactory getCheckpointRecoveryFactory() throws Exception {
+ return new ZooKeeperCheckpointRecoveryFactory(client, configuration);
+ }
+
+ @Override
+ public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
+ return ZooKeeperUtils.createSubmittedJobGraphs(client, configuration);
+ }
+
+ private static String getPathSuffixForJob(final JobID jobID) {
+ return String.format("/job-managers/%s", jobID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/StandaloneSubmittedJobGraphStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/StandaloneSubmittedJobGraphStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/StandaloneSubmittedJobGraphStore.java
index 3041cde..00df935 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/StandaloneSubmittedJobGraphStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/StandaloneSubmittedJobGraphStore.java
@@ -62,4 +62,9 @@ public class StandaloneSubmittedJobGraphStore implements SubmittedJobGraphStore
public List<SubmittedJobGraph> recoverJobGraphs() throws Exception {
return Collections.emptyList();
}
+
+ @Override
+ public boolean contains(JobID jobId) throws Exception {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/SubmittedJobGraphStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/SubmittedJobGraphStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/SubmittedJobGraphStore.java
index bd628cd..4d544ae 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/SubmittedJobGraphStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/SubmittedJobGraphStore.java
@@ -64,6 +64,14 @@ public interface SubmittedJobGraphStore {
void removeJobGraph(JobID jobId) throws Exception;
/**
+ * Check whether the given {@link JobID} is exist.
+ *
+ * <p>It's also a flag indicates whether we should recover this job before we can do anything else, since all
+ * global terminated job will be removed from this store.
+ */
+ boolean contains(final JobID jobId) throws Exception;
+
+ /**
* A listener for {@link SubmittedJobGraph} instances. This is used to react to races between
* multiple running {@link SubmittedJobGraphStore} instances (on multiple job managers).
*/
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStore.java
index ec05f1e..92093c5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/ZooKeeperSubmittedJobGraphStore.java
@@ -266,6 +266,13 @@ public class ZooKeeperSubmittedJobGraphStore implements SubmittedJobGraphStore {
}
}
+ @Override
+ public boolean contains(JobID jobId) throws Exception {
+ checkNotNull(jobId, "Job ID");
+ String path = getPathForJob(jobId);
+ return jobGraphsInZooKeeper.exists(path) != -1;
+ }
+
/**
* Monitors ZooKeeper for changes.
*
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
index 6944d85..a096932 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
@@ -21,20 +21,18 @@ package org.apache.flink.runtime.jobmaster;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobmanager.OnCompletionActions;
+import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
import org.apache.flink.runtime.leaderelection.LeaderContender;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.rpc.RpcService;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.UUID;
-import java.util.concurrent.Executor;
/**
* The runner for the job manager. It deals with job level leader election and make underlying job manager
@@ -52,11 +50,8 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
private final OnCompletionActions toNotify;
- /** The execution context which is used to execute futures */
- private final Executor executionContext;
-
- // TODO: use this to decide whether the job is finished by other
- private final CheckpointRecoveryFactory checkpointRecoveryFactory;
+ /** Used to check whether a job needs to be run */
+ private final SubmittedJobGraphStore submittedJobGraphStore;
/** Leader election for this job */
private final LeaderElectionService leaderElectionService;
@@ -87,9 +82,8 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
{
this.jobGraph = jobGraph;
this.toNotify = toNotify;
- this.executionContext = rpcService.getExecutor();
- this.checkpointRecoveryFactory = haServices.getCheckpointRecoveryFactory();
- this.leaderElectionService = haServices.getJobMasterLeaderElectionService(jobGraph.getJobID());
+ this.submittedJobGraphStore = haServices.getSubmittedJobGraphStore();
+ this.leaderElectionService = haServices.getJobManagerLeaderElectionService(jobGraph.getJobID());
this.jobManager = new JobMaster(
jobGraph, configuration, rpcService, haServices,
@@ -271,7 +265,7 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions {
@VisibleForTesting
boolean isJobFinishedByOthers() {
- // TODO
+ // TODO: Fix
return false;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 1e01c55..e67a167 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -24,6 +24,8 @@ import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.io.InputSplit;
+import org.apache.flink.core.io.InputSplitAssigner;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
@@ -35,23 +37,32 @@ import org.apache.flink.runtime.checkpoint.stats.DisabledCheckpointStatsTracker;
import org.apache.flink.runtime.checkpoint.stats.SimpleCheckpointStatsTracker;
import org.apache.flink.runtime.client.JobExecutionException;
import org.apache.flink.runtime.client.JobSubmissionException;
+import org.apache.flink.runtime.client.SerializedJobExecutionResult;
+import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager;
+import org.apache.flink.runtime.executiongraph.Execution;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.executiongraph.ExecutionGraph;
import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
import org.apache.flink.runtime.executiongraph.restart.RestartStrategy;
import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.instance.Slot;
+import org.apache.flink.runtime.io.network.PartitionState;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
+import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertex;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.jsonplan.JsonPlanGenerator;
import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
import org.apache.flink.runtime.jobmanager.OnCompletionActions;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
+import org.apache.flink.runtime.jobmaster.message.NextInputSplit;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
-import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup;
import org.apache.flink.runtime.registration.RegisteredRpcConnection;
import org.apache.flink.runtime.registration.RegistrationResponse;
@@ -61,13 +72,18 @@ import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
+import org.apache.flink.runtime.util.SerializedThrowable;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.SerializedValue;
import org.slf4j.Logger;
import scala.concurrent.ExecutionContext$;
import scala.concurrent.duration.FiniteDuration;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Executor;
@@ -491,9 +507,12 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
* @return Acknowledge the task execution state update
*/
@RpcMethod
- public Acknowledge updateTaskExecutionState(TaskExecutionState taskExecutionState) {
- System.out.println("TaskExecutionState: " + taskExecutionState);
- return Acknowledge.get();
+ public boolean updateTaskExecutionState(final TaskExecutionState taskExecutionState) {
+ if (taskExecutionState == null) {
+ return false;
+ } else {
+ return executionGraph.updateState(taskExecutionState);
+ }
}
//----------------------------------------------------------------------------------------------\u2028
@@ -511,6 +530,140 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
});
}
+ @RpcMethod
+ public NextInputSplit requestNextInputSplit(
+ final JobVertexID vertexID,
+ final ExecutionAttemptID executionAttempt) throws Exception
+ {
+ final Execution execution = executionGraph.getRegisteredExecutions().get(executionAttempt);
+ if (execution == null) {
+ // can happen when JobManager had already unregistered this execution upon on task failure,
+ // but TaskManager get some delay to aware of that situation
+ if (log.isDebugEnabled()) {
+ log.debug("Can not find Execution for attempt {}.", executionAttempt);
+ }
+ // but we should TaskManager be aware of this
+ throw new Exception("Can not find Execution for attempt " + executionAttempt);
+ }
+
+ final ExecutionJobVertex vertex = executionGraph.getJobVertex(vertexID);
+ if (vertex == null) {
+ log.error("Cannot find execution vertex for vertex ID {}.", vertexID);
+ throw new Exception("Cannot find execution vertex for vertex ID " + vertexID);
+ }
+
+ final InputSplitAssigner splitAssigner = vertex.getSplitAssigner();
+ if (splitAssigner == null) {
+ log.error("No InputSplitAssigner for vertex ID {}.", vertexID);
+ throw new Exception("No InputSplitAssigner for vertex ID " + vertexID);
+ }
+
+ final Slot slot = execution.getAssignedResource();
+ final int taskId = execution.getVertex().getParallelSubtaskIndex();
+ final String host = slot != null ? slot.getTaskManagerLocation().getHostname() : null;
+ final InputSplit nextInputSplit = splitAssigner.getNextInputSplit(host, taskId);
+
+ if (log.isDebugEnabled()) {
+ log.debug("Send next input split {}.", nextInputSplit);
+ }
+
+ try {
+ final byte[] serializedInputSplit = InstantiationUtil.serializeObject(nextInputSplit);
+ return new NextInputSplit(serializedInputSplit);
+ } catch (Exception ex) {
+ log.error("Could not serialize the next input split of class {}.", nextInputSplit.getClass(), ex);
+ IOException reason = new IOException("Could not serialize the next input split of class " +
+ nextInputSplit.getClass() + ".", ex);
+ vertex.fail(reason);
+ throw reason;
+ }
+ }
+
+ @RpcMethod
+ public PartitionState requestPartitionState(
+ final ResultPartitionID partitionId,
+ final ExecutionAttemptID taskExecutionId,
+ final IntermediateDataSetID taskResultId)
+ {
+ final Execution execution = executionGraph.getRegisteredExecutions().get(partitionId.getProducerId());
+ final ExecutionState state = execution != null ? execution.getState() : null;
+ return new PartitionState(taskResultId, partitionId.getPartitionId(), state);
+ }
+
+ @RpcMethod
+ public void scheduleOrUpdateConsumers(final ResultPartitionID partitionID) {
+ executionGraph.scheduleOrUpdateConsumers(partitionID);
+ }
+
+ //----------------------------------------------------------------------------------------------
+ // Internal methods
+ //----------------------------------------------------------------------------------------------
+
+ // TODO - wrap this as StatusListenerMessenger's callback with rpc main thread
+ private void jobStatusChanged(final JobStatus newJobStatus, long timestamp, final Throwable error) {
+ final JobID jobID = executionGraph.getJobID();
+ final String jobName = executionGraph.getJobName();
+ log.info("Status of job {} ({}) changed to {}.", jobID, jobName, newJobStatus, error);
+
+ if (newJobStatus.isGloballyTerminalState()) {
+ // TODO set job end time in JobInfo
+
+ /*
+ TODO
+ if (jobInfo.sessionAlive) {
+ jobInfo.setLastActive()
+ val lastActivity = jobInfo.lastActive
+ context.system.scheduler.scheduleOnce(jobInfo.sessionTimeout seconds) {
+ // remove only if no activity occurred in the meantime
+ if (lastActivity == jobInfo.lastActive) {
+ self ! decorateMessage(RemoveJob(jobID, removeJobFromStateBackend = true))
+ }
+ }(context.dispatcher)
+ } else {
+ self ! decorateMessage(RemoveJob(jobID, removeJobFromStateBackend = true))
+ }
+ */
+
+ if (newJobStatus == JobStatus.FINISHED) {
+ try {
+ final Map<String, SerializedValue<Object>> accumulatorResults =
+ executionGraph.getAccumulatorsSerialized();
+ final SerializedJobExecutionResult result = new SerializedJobExecutionResult(
+ jobID, 0, accumulatorResults // TODO get correct job duration
+ );
+ jobCompletionActions.jobFinished(result.toJobExecutionResult(userCodeLoader));
+ } catch (Exception e) {
+ log.error("Cannot fetch final accumulators for job {} ({})", jobName, jobID, e);
+ final JobExecutionException exception = new JobExecutionException(
+ jobID, "Failed to retrieve accumulator results.", e);
+ // TODO should we also notify client?
+ jobCompletionActions.jobFailed(exception);
+ }
+ }
+ else if (newJobStatus == JobStatus.CANCELED) {
+ final Throwable unpackedError = SerializedThrowable.get(error, userCodeLoader);
+ final JobExecutionException exception = new JobExecutionException(
+ jobID, "Job was cancelled.", unpackedError);
+ // TODO should we also notify client?
+ jobCompletionActions.jobFailed(exception);
+ }
+ else if (newJobStatus == JobStatus.FAILED) {
+ final Throwable unpackedError = SerializedThrowable.get(error, userCodeLoader);
+ final JobExecutionException exception = new JobExecutionException(
+ jobID, "Job execution failed.", unpackedError);
+ // TODO should we also notify client?
+ jobCompletionActions.jobFailed(exception);
+ }
+ else {
+ final JobExecutionException exception = new JobExecutionException(
+ jobID, newJobStatus + " is not a terminal state.");
+ // TODO should we also notify client?
+ jobCompletionActions.jobFailed(exception);
+ throw new RuntimeException(exception);
+ }
+ }
+ }
+
private void notifyOfNewResourceManagerLeader(
final String resourceManagerAddress, final UUID resourceManagerLeaderId)
{
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
index 6587ccb..686a3f3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
@@ -19,7 +19,15 @@
package org.apache.flink.runtime.jobmaster;
import org.apache.flink.runtime.concurrent.Future;
-import org.apache.flink.runtime.messages.Acknowledge;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+import org.apache.flink.runtime.executiongraph.ExecutionVertex;
+import org.apache.flink.runtime.io.network.PartitionState;
+import org.apache.flink.runtime.io.network.partition.ResultPartition;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
+import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobmaster.message.NextInputSplit;
import org.apache.flink.runtime.rpc.RpcGateway;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
@@ -47,7 +55,47 @@ public interface JobMasterGateway extends RpcGateway {
* Updates the task execution state for a given task.
*
* @param taskExecutionState New task execution state for a given task
- * @return Future acknowledge of the task execution state update
+ * @return Future flag of the task execution state update result
*/
- Future<Acknowledge> updateTaskExecutionState(TaskExecutionState taskExecutionState);
+ Future<Boolean> updateTaskExecutionState(TaskExecutionState taskExecutionState);
+
+ /**
+ * Requesting next input split for the {@link ExecutionJobVertex}. The next input split is sent back to the sender
+ * as a {@link NextInputSplit} message.
+ *
+ * @param vertexID The job vertex id
+ * @param executionAttempt The execution attempt id
+ * @return The future of the input split. If there is no further input split, will return an empty object.
+ * @throws Exception if some error occurred or information mismatch.
+ */
+ Future<NextInputSplit> requestNextInputSplit(
+ final JobVertexID vertexID,
+ final ExecutionAttemptID executionAttempt) throws Exception;
+
+ /**
+ * Requests the current state of the partition.
+ * The state of a partition is currently bound to the state of the producing execution.
+ *
+ * @param partitionId The partition ID of the partition to request the state of.
+ * @param taskExecutionId The execution attempt ID of the task requesting the partition state.
+ * @param taskResultId The input gate ID of the task requesting the partition state.
+ * @return The future of the partition state
+ */
+ Future<PartitionState> requestPartitionState(
+ final ResultPartitionID partitionId,
+ final ExecutionAttemptID taskExecutionId,
+ final IntermediateDataSetID taskResultId);
+
+ /**
+ * Notifies the JobManager about available data for a produced partition.
+ * <p>
+ * There is a call to this method for each {@link ExecutionVertex} instance once per produced
+ * {@link ResultPartition} instance, either when first producing data (for pipelined executions)
+ * or when all data has been produced (for staged executions).
+ * <p>
+ * The JobManager then can decide when to schedule the partition consumers of the given session.
+ *
+ * @param partitionID The partition which has already produced data
+ */
+ void scheduleOrUpdateConsumers(final ResultPartitionID partitionID);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/NextInputSplit.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/NextInputSplit.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/NextInputSplit.java
new file mode 100644
index 0000000..fe511ed
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/message/NextInputSplit.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster.message;
+
+import java.io.Serializable;
+
+/**
+ * Contains the next input split for a task.
+ */
+public class NextInputSplit implements Serializable {
+
+ private static final long serialVersionUID = -1355784074565856240L;
+
+ private final byte[] splitData;
+
+ public NextInputSplit(final byte[] splitData) {
+ this.splitData = splitData;
+ }
+
+ public byte[] getSplitData() {
+ return splitData;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index f695de4..f45afa3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -129,7 +129,7 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
try {
leaderElectionService.stop();
for (JobID jobID : jobMasterGateways.keySet()) {
- highAvailabilityServices.getJobMasterLeaderRetriever(jobID).stop();
+ highAvailabilityServices.getJobManagerLeaderRetriever(jobID).stop();
}
super.shutDown();
} catch (Throwable e) {
@@ -179,7 +179,7 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
final LeaderConnectionInfo jobMasterLeaderInfo;
try {
jobMasterLeaderInfo = LeaderRetrievalUtils.retrieveLeaderConnectionInfo(
- highAvailabilityServices.getJobMasterLeaderRetriever(jobID), new FiniteDuration(5, TimeUnit.SECONDS));
+ highAvailabilityServices.getJobManagerLeaderRetriever(jobID), new FiniteDuration(5, TimeUnit.SECONDS));
} catch (Exception e) {
log.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
throw new Exception("Failed to retrieve JobMasterLeaderRetriever");
@@ -203,7 +203,7 @@ public abstract class ResourceManager<WorkerType extends ResourceIDRetrievable>
if (!jobMasterLeaderRetrievalListeners.containsKey(jobID)) {
JobMasterLeaderListener jobMasterLeaderListener = new JobMasterLeaderListener(jobID);
try {
- LeaderRetrievalService jobMasterLeaderRetriever = highAvailabilityServices.getJobMasterLeaderRetriever(jobID);
+ LeaderRetrievalService jobMasterLeaderRetriever = highAvailabilityServices.getJobManagerLeaderRetriever(jobID);
jobMasterLeaderRetriever.start(jobMasterLeaderListener);
} catch (Exception e) {
log.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java
index a9887a6..e9777a3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java
@@ -161,12 +161,45 @@ public class ZooKeeperUtils {
*/
public static ZooKeeperLeaderRetrievalService createLeaderRetrievalService(
Configuration configuration) {
- CuratorFramework client = startCuratorFramework(configuration);
+ {
+ final CuratorFramework client = startCuratorFramework(configuration);
+ return createLeaderRetrievalService(client, configuration);
+ }
+
+ /**
+ * Creates a {@link ZooKeeperLeaderRetrievalService} instance.
+ *
+ * @param client The {@link CuratorFramework} ZooKeeper client to use
+ * @param configuration {@link Configuration} object containing the configuration values
+ * @return {@link ZooKeeperLeaderRetrievalService} instance.
+ * @throws Exception
+ */
+ public static ZooKeeperLeaderRetrievalService createLeaderRetrievalService(
+ final CuratorFramework client,
+ final Configuration configuration) throws Exception
+ {
+ return createLeaderRetrievalService(client, configuration, "");
+ }
+
+ /**
+ * Creates a {@link ZooKeeperLeaderRetrievalService} instance.
+ *
+ * @param client The {@link CuratorFramework} ZooKeeper client to use
+ * @param configuration {@link Configuration} object containing the configuration values
+ * @param pathSuffix The path suffix which we want to append
+ * @return {@link ZooKeeperLeaderRetrievalService} instance.
+ * @throws Exception
+ */
+ public static ZooKeeperLeaderRetrievalService createLeaderRetrievalService(
+ final CuratorFramework client,
+ final Configuration configuration,
+ final String pathSuffix) throws Exception
+ {
String leaderPath = ConfigurationUtil.getStringWithDeprecatedKeys(
- configuration,
- ConfigConstants.HA_ZOOKEEPER_LEADER_PATH,
- ConfigConstants.DEFAULT_ZOOKEEPER_LEADER_PATH,
- ConfigConstants.ZOOKEEPER_LEADER_PATH);
+ configuration,
+ ConfigConstants.HA_ZOOKEEPER_LEADER_PATH,
+ ConfigConstants.DEFAULT_ZOOKEEPER_LEADER_PATH,
+ ConfigConstants.ZOOKEEPER_LEADER_PATH) + pathSuffix;
return new ZooKeeperLeaderRetrievalService(client, leaderPath);
}
@@ -197,16 +230,33 @@ public class ZooKeeperUtils {
CuratorFramework client,
Configuration configuration) {
- String latchPath = ConfigurationUtil.getStringWithDeprecatedKeys(
- configuration,
- ConfigConstants.HA_ZOOKEEPER_LATCH_PATH,
- ConfigConstants.DEFAULT_ZOOKEEPER_LATCH_PATH,
- ConfigConstants.ZOOKEEPER_LATCH_PATH);
- String leaderPath = ConfigurationUtil.getStringWithDeprecatedKeys(
- configuration,
- ConfigConstants.HA_ZOOKEEPER_LEADER_PATH,
- ConfigConstants.DEFAULT_ZOOKEEPER_LEADER_PATH,
- ConfigConstants.ZOOKEEPER_LEADER_PATH);
+ return createLeaderElectionService(client, configuration, "");
+ }
+
+ /**
+ * Creates a {@link ZooKeeperLeaderElectionService} instance.
+ *
+ * @param client The {@link CuratorFramework} ZooKeeper client to use
+ * @param configuration {@link Configuration} object containing the configuration values
+ * @param pathSuffix The path suffix which we want to append
+ * @return {@link ZooKeeperLeaderElectionService} instance.
+ * @throws Exception
+ */
+ public static ZooKeeperLeaderElectionService createLeaderElectionService(
+ final CuratorFramework client,
+ final Configuration configuration,
+ final String pathSuffix) throws Exception
+ {
+ final String latchPath = ConfigurationUtil.getStringWithDeprecatedKeys(
+ configuration,
+ ConfigConstants.HA_ZOOKEEPER_LATCH_PATH,
+ ConfigConstants.DEFAULT_ZOOKEEPER_LATCH_PATH,
+ ConfigConstants.ZOOKEEPER_LATCH_PATH) + pathSuffix;
+ final String leaderPath = ConfigurationUtil.getStringWithDeprecatedKeys(
+ configuration,
+ ConfigConstants.HA_ZOOKEEPER_LEADER_PATH,
+ ConfigConstants.DEFAULT_ZOOKEEPER_LEADER_PATH,
+ ConfigConstants.ZOOKEEPER_LEADER_PATH) + pathSuffix;
return new ZooKeeperLeaderElectionService(client, latchPath, leaderPath);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 0701f3a..4007921 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -45,6 +45,8 @@ import org.apache.flink.runtime.filecache.FileCache
import org.apache.flink.runtime.instance.{AkkaActorGateway, HardwareDescription, InstanceID}
import org.apache.flink.runtime.io.disk.iomanager.IOManager
import org.apache.flink.runtime.io.network.NetworkEnvironment
+import org.apache.flink.runtime.io.network.netty.PartitionStateChecker
+import org.apache.flink.runtime.io.network.partition.ResultPartitionConsumableNotifier
import org.apache.flink.runtime.leaderretrieval.{LeaderRetrievalListener, LeaderRetrievalService}
import org.apache.flink.runtime.memory.MemoryManager
import org.apache.flink.runtime.messages.{Acknowledge, StackTraceSampleResponse}
@@ -919,12 +921,12 @@ class TaskManager(
val partitionStateChecker = new ActorGatewayPartitionStateChecker(
jobManagerGateway,
- config.timeout)
+ new FiniteDuration(config.getTimeout().toMilliseconds, TimeUnit.MILLISECONDS))
val resultPartitionConsumableNotifier = new ActorGatewayResultPartitionConsumableNotifier(
context.dispatcher,
jobManagerGateway,
- config.timeout)
+ new FiniteDuration(config.getTimeout().toMilliseconds, TimeUnit.MILLISECONDS))
connectionUtils = Some(
(checkpointResponder,
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
index 1a5450d..faf69cc 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
@@ -36,7 +36,7 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
private ConcurrentHashMap<JobID, LeaderRetrievalService> jobMasterLeaderRetrievers = new ConcurrentHashMap<>();
- private volatile LeaderElectionService jobMasterLeaderElectionService;
+ private ConcurrentHashMap<JobID, LeaderElectionService> jobManagerLeaderElectionServices = new ConcurrentHashMap<>();
private volatile LeaderElectionService resourceManagerLeaderElectionService;
@@ -56,8 +56,8 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
this.jobMasterLeaderRetrievers.put(jobID, jobMasterLeaderRetriever);
}
- public void setJobMasterLeaderElectionService(LeaderElectionService leaderElectionService) {
- this.jobMasterLeaderElectionService = leaderElectionService;
+ public void setJobMasterLeaderElectionService(JobID jobID, LeaderElectionService leaderElectionService) {
+ this.jobManagerLeaderElectionServices.put(jobID, leaderElectionService);
}
public void setResourceManagerLeaderElectionService(LeaderElectionService leaderElectionService) {
@@ -87,7 +87,7 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
}
@Override
- public LeaderRetrievalService getJobMasterLeaderRetriever(JobID jobID) throws Exception {
+ public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception {
LeaderRetrievalService service = this.jobMasterLeaderRetrievers.get(jobID);
if (service != null) {
return service;
@@ -97,24 +97,24 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
}
@Override
- public LeaderElectionService getJobMasterLeaderElectionService(JobID jobID) throws Exception {
- LeaderElectionService service = jobMasterLeaderElectionService;
+ public LeaderElectionService getResourceManagerLeaderElectionService() throws Exception {
+ LeaderElectionService service = resourceManagerLeaderElectionService;
if (service != null) {
return service;
} else {
- throw new IllegalStateException("JobMasterLeaderElectionService has not been set");
+ throw new IllegalStateException("ResourceManagerLeaderElectionService has not been set");
}
}
@Override
- public LeaderElectionService getResourceManagerLeaderElectionService() throws Exception {
- LeaderElectionService service = resourceManagerLeaderElectionService;
+ public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) throws Exception {
+ LeaderElectionService service = this.jobManagerLeaderElectionServices.get(jobID);
if (service != null) {
return service;
} else {
- throw new IllegalStateException("ResourceManagerLeaderElectionService has not been set");
+ throw new IllegalStateException("JobMasterLeaderElectionService has not been set");
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
index b195858..24a7134 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
@@ -404,7 +404,8 @@ public class JobManagerHARecoveryTest {
storedJobs.remove(jobId);
}
- boolean contains(JobID jobId) {
+ @Override
+ public boolean contains(JobID jobId) {
return storedJobs.containsKey(jobId);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
index bfe5f55..3a769bb 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
@@ -24,6 +24,7 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobmanager.OnCompletionActions;
+import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.rpc.RpcService;
import org.junit.After;
@@ -57,6 +58,8 @@ public class JobManagerRunnerMockTest {
private LeaderElectionService leaderElectionService;
+ private SubmittedJobGraphStore submittedJobGraphStore;
+
private TestingOnCompletionActions jobCompletion;
@Before
@@ -72,8 +75,12 @@ public class JobManagerRunnerMockTest {
leaderElectionService = mock(LeaderElectionService.class);
when(leaderElectionService.hasLeadership()).thenReturn(true);
+ submittedJobGraphStore = mock(SubmittedJobGraphStore.class);
+ when(submittedJobGraphStore.contains(any(JobID.class))).thenReturn(true);
+
HighAvailabilityServices haServices = mock(HighAvailabilityServices.class);
- when(haServices.getJobMasterLeaderElectionService(any(JobID.class))).thenReturn(leaderElectionService);
+ when(haServices.getJobManagerLeaderElectionService(any(JobID.class))).thenReturn(leaderElectionService);
+ when(haServices.getSubmittedJobGraphStore()).thenReturn(submittedJobGraphStore);
runner = PowerMockito.spy(new JobManagerRunner(
new JobGraph("test"),
@@ -127,7 +134,7 @@ public class JobManagerRunnerMockTest {
public void testJobFinishedByOtherBeforeGrantLeadership() throws Exception {
runner.start();
- when(runner.isJobFinishedByOthers()).thenReturn(true);
+ when(submittedJobGraphStore.contains(any(JobID.class))).thenReturn(false);
runner.grantLeadership(UUID.randomUUID());
// runner should shutdown automatic and informed the job completion
http://git-wip-us.apache.org/repos/asf/flink/blob/c7e1d8f6/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
index e3018c9..805ea71 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
@@ -237,7 +237,7 @@ public class SlotProtocolTest extends TestLogger {
testingHA.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);
final TestingLeaderElectionService jmLeaderElectionService = new TestingLeaderElectionService();
- testingHA.setJobMasterLeaderElectionService(jmLeaderElectionService);
+ testingHA.setJobMasterLeaderElectionService(jobID, jmLeaderElectionService);
final TestingLeaderRetrievalService jmLeaderRetrievalService = new TestingLeaderRetrievalService(jmAddress, jmID);
testingHA.setJobMasterLeaderRetriever(jobID, jmLeaderRetrievalService);
[44/50] [abbrv] flink git commit: [FLINK-4836] [cluster management]
Start ResourceManager in MiniCluster
Posted by tr...@apache.org.
[FLINK-4836] [cluster management] Start ResourceManager in MiniCluster
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/7ba5b866
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/7ba5b866
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/7ba5b866
Branch: refs/heads/flip-6
Commit: 7ba5b8669c6ada8d331282818ae2e5a79f1ee092
Parents: 2e067fc
Author: Stephan Ewen <se...@apache.org>
Authored: Mon Oct 17 17:02:33 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:34 2016 +0100
----------------------------------------------------------------------
.../flink/runtime/minicluster/MiniCluster.java | 126 +++++++++++++++----
.../minicluster/MiniClusterJobDispatcher.java | 4 +-
2 files changed, 102 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/7ba5b866/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
index 1ffcd12..d63f9a7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
@@ -32,11 +32,18 @@ import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
+import org.apache.flink.runtime.resourcemanager.ResourceManager;
+import org.apache.flink.runtime.resourcemanager.StandaloneResourceManager;
+import org.apache.flink.runtime.resourcemanager.slotmanager.DefaultSlotManager;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
import org.apache.flink.runtime.taskexecutor.TaskManagerRunner;
import org.apache.flink.util.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import javax.annotation.concurrent.GuardedBy;
import java.util.UUID;
@@ -48,13 +55,15 @@ import static org.apache.flink.util.Preconditions.checkState;
public class MiniCluster {
+ private static final Logger LOG = LoggerFactory.getLogger(MiniCluster.class);
+
/** The lock to guard startup / shutdown / manipulation methods */
private final Object lock = new Object();
/** The configuration for this mini cluster */
private final MiniClusterConfiguration config;
- @GuardedBy("lock")
+ @GuardedBy("lock")
private MetricRegistry metricRegistry;
@GuardedBy("lock")
@@ -73,6 +82,9 @@ public class MiniCluster {
private HighAvailabilityServices haServices;
@GuardedBy("lock")
+ private ResourceManager<?>[] resourceManagers;
+
+ @GuardedBy("lock")
private TaskManagerRunner[] taskManagerRunners;
@GuardedBy("lock")
@@ -98,6 +110,7 @@ public class MiniCluster {
}
/**
+ * Creates a new Flink mini cluster based on the given configuration.
*
* @param config The configuration for the mini cluster
*/
@@ -149,6 +162,9 @@ public class MiniCluster {
synchronized (lock) {
checkState(!running, "FlinkMiniCluster is already running");
+ LOG.info("Starting Flink Mini Cluster");
+ LOG.debug("Using configuration {}", config);
+
final Configuration configuration = new UnmodifiableConfiguration(config.getConfiguration());
final Time rpcTimeout = config.getRpcTimeout();
final int numJobManagers = config.getNumJobManagers();
@@ -210,13 +226,21 @@ public class MiniCluster {
}
// create the high-availability services
+ LOG.info("Starting high-availability services");
haServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(configuration);
- // bring up the task managers for the mini cluster
+ // bring up the ResourceManager(s)
+ LOG.info("Starting {} ResourceManger(s)", numResourceManagers);
+ resourceManagers = startResourceManagers(
+ configuration, haServices, metricRegistry, numResourceManagers, resourceManagerRpcServices);
+
+ // bring up the TaskManager(s) for the mini cluster
+ LOG.info("Starting {} TaskManger(s)", numTaskManagers);
taskManagerRunners = startTaskManagers(
configuration, haServices, metricRegistry, numTaskManagers, taskManagerRpcServices);
// bring up the dispatcher that launches JobManagers when jobs submitted
+ LOG.info("Starting job dispatcher for {} JobManger(s)", numJobManagers);
jobDispatcher = new MiniClusterJobDispatcher(
configuration, haServices, metricRegistry, numJobManagers, jobManagerRpcServices);
}
@@ -232,6 +256,8 @@ public class MiniCluster {
// now officially mark this as running
running = true;
+
+ LOG.info("Flink Mini Cluster started successfully");
}
}
@@ -247,11 +273,13 @@ public class MiniCluster {
public void shutdown() throws Exception {
synchronized (lock) {
if (running) {
+ LOG.info("Shutting down Flink Mini Cluster");
try {
shutdownInternally();
} finally {
running = false;
}
+ LOG.info("Flink Mini Cluster is shut down");
}
}
}
@@ -270,11 +298,34 @@ public class MiniCluster {
try {
jobDispatcher.shutdown();
} catch (Exception e) {
- exception = firstOrSuppressed(e, exception);
+ exception = e;
}
jobDispatcher = null;
}
+ if (resourceManagers != null) {
+ for (ResourceManager<?> rm : resourceManagers) {
+ if (rm != null) {
+ try {
+ rm.shutDown();
+ } catch (Throwable t) {
+ exception = firstOrSuppressed(t, exception);
+ }
+ }
+ }
+ resourceManagers = null;
+ }
+
+ // shut down the RpcServices
+ exception = shutDownRpc(commonRpcService, exception);
+ exception = shutDownRpcs(jobManagerRpcServices, exception);
+ exception = shutDownRpcs(taskManagerRpcServices, exception);
+ exception = shutDownRpcs(resourceManagerRpcServices, exception);
+ commonRpcService = null;
+ jobManagerRpcServices = null;
+ taskManagerRpcServices = null;
+ resourceManagerRpcServices = null;
+
// shut down high-availability services
if (haServices != null) {
try {
@@ -285,24 +336,6 @@ public class MiniCluster {
haServices = null;
}
- // shut down the RpcServices
- if (commonRpcService != null) {
- exception = shutDownRpc(commonRpcService, exception);
- commonRpcService = null;
- }
- if (jobManagerRpcServices != null) {
- for (RpcService service : jobManagerRpcServices) {
- exception = shutDownRpc(service, exception);
- }
- jobManagerRpcServices = null;
- }
- if (taskManagerRpcServices != null) {
- for (RpcService service : taskManagerRpcServices) {
- exception = shutDownRpc(service, exception);
- }
- taskManagerRpcServices = null;
- }
-
// metrics shutdown
if (metricRegistry != null) {
metricRegistry.shutdown();
@@ -402,6 +435,28 @@ public class MiniCluster {
return new AkkaRpcService(actorSystem, askTimeout);
}
+ protected ResourceManager<?>[] startResourceManagers(
+ Configuration configuration,
+ HighAvailabilityServices haServices,
+ MetricRegistry metricRegistry,
+ int numResourceManagers,
+ RpcService[] resourceManagerRpcServices) throws Exception {
+
+ final StandaloneResourceManager[] resourceManagers = new StandaloneResourceManager[numResourceManagers];
+ final SlotManagerFactory slotManagerFactory = new DefaultSlotManager.Factory();
+
+ for (int i = 0; i < numResourceManagers; i++) {
+ resourceManagers[i] = new StandaloneResourceManager(
+ resourceManagerRpcServices[i],
+ haServices,
+ slotManagerFactory);
+
+ resourceManagers[i].start();
+ }
+
+ return resourceManagers;
+ }
+
protected TaskManagerRunner[] startTaskManagers(
Configuration configuration,
HighAvailabilityServices haServices,
@@ -429,15 +484,34 @@ public class MiniCluster {
// ------------------------------------------------------------------------
private static Throwable shutDownRpc(RpcService rpcService, Throwable priorException) {
- try {
- if (rpcService != null) {
+ if (rpcService != null) {
+ try {
rpcService.stopService();
}
- return priorException;
+ catch (Throwable t) {
+ return firstOrSuppressed(t, priorException);
+ }
}
- catch (Throwable t) {
- return firstOrSuppressed(t, priorException);
+
+ return priorException;
+ }
+
+ private static Throwable shutDownRpcs(RpcService[] rpcServices, Throwable priorException) {
+ if (rpcServices != null) {
+ Throwable exception = priorException;
+
+ for (RpcService service : rpcServices) {
+ try {
+ if (service != null) {
+ service.stopService();
+ }
+ }
+ catch (Throwable t) {
+ exception = firstOrSuppressed(t, exception);
+ }
+ }
}
+ return priorException;
}
private static MiniClusterConfiguration createConfig(Configuration cfg, boolean singleRpcService) {
http://git-wip-us.apache.org/repos/asf/flink/blob/7ba5b866/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterJobDispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterJobDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterJobDispatcher.java
index d0df293..8ac8eba 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterJobDispatcher.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterJobDispatcher.java
@@ -176,7 +176,7 @@ public class MiniClusterJobDispatcher {
public void runDetached(JobGraph job) throws JobExecutionException {
checkNotNull(job);
- LOG.info("Received job for detached execution {} ({})", job.getName(), job.getJobID());
+ LOG.info("Received job for detached execution: {} ({})", job.getName(), job.getJobID());
synchronized (lock) {
checkState(!shutdown, "mini cluster is shut down");
@@ -201,7 +201,7 @@ public class MiniClusterJobDispatcher {
public JobExecutionResult runJobBlocking(JobGraph job) throws JobExecutionException, InterruptedException {
checkNotNull(job);
- LOG.info("Received job for blocking execution {} ({})", job.getName(), job.getJobID());
+ LOG.info("Received job for blocking execution: {} ({})", job.getName(), job.getJobID());
final BlockingJobSync sync = new BlockingJobSync(job.getJobID(), numJobManagers);
synchronized (lock) {
[24/50] [abbrv] flink git commit: [FLINK-4360] [tm] Implement TM ->
JM registration logic
Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/SlotAllocationException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/SlotAllocationException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/SlotAllocationException.java
new file mode 100644
index 0000000..66d8102
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/exceptions/SlotAllocationException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskexecutor.exceptions;
+
+/**
+ * Exception indicating that the slot allocation on the task manager failed.
+ */
+public class SlotAllocationException extends TaskManagerException {
+
+ private static final long serialVersionUID = -4764932098204266773L;
+
+ public SlotAllocationException(String message) {
+ super(message);
+ }
+
+ public SlotAllocationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public SlotAllocationException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
index 42cb919..88123b4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/slot/TaskSlotTable.java
@@ -70,10 +70,7 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
/** Interface for slot actions, such as freeing them or timing them out */
private SlotActions slotActions;
-
- /** The timeout for allocated slots */
- private Time slotTimeout;
-
+
/** Whether the table has been started */
private boolean started;
@@ -104,7 +101,6 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
slotsPerJob = new HashMap<>(4);
slotActions = null;
- slotTimeout = null;
started = false;
}
@@ -112,11 +108,9 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
* Start the task slot table with the given slot actions and slot timeout value.
*
* @param initialSlotActions to use for slot actions
- * @param initialSlotTimeout to use for slot timeouts
*/
- public void start(SlotActions initialSlotActions, Time initialSlotTimeout) {
+ public void start(SlotActions initialSlotActions) {
this.slotActions = Preconditions.checkNotNull(initialSlotActions);
- this.slotTimeout = Preconditions.checkNotNull(initialSlotTimeout);
timerService.start(this);
@@ -129,7 +123,6 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
public void stop() {
started = false;
timerService.stop();
- slotTimeout = null;
slotActions = null;
}
@@ -144,9 +137,10 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
* @param index of the task slot to allocate
* @param jobId to allocate the task slot for
* @param allocationId identifying the allocation
+ * @param slotTimeout until the slot times out
* @return True if the task slot could be allocated; otherwise false
*/
- public boolean allocateSlot(int index, JobID jobId, AllocationID allocationId) {
+ public boolean allocateSlot(int index, JobID jobId, AllocationID allocationId, Time slotTimeout) {
checkInit();
TaskSlot taskSlot = taskSlots.get(index);
@@ -180,7 +174,7 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
*
* @param allocationId to identify the task slot to mark as active
* @throws SlotNotFoundException if the slot could not be found for the given allocation id
- * @return True if the slot could be marked active
+ * @return True if the slot could be marked active; otherwise false
*/
public boolean markSlotActive(AllocationID allocationId) throws SlotNotFoundException {
checkInit();
@@ -190,6 +184,8 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
if (taskSlot != null) {
if (taskSlot.markActive()) {
// unregister a potential timeout
+ LOG.info("Activate slot {}.", allocationId);
+
timerService.unregisterTimeout(allocationId);
return true;
@@ -206,10 +202,11 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
* then a {@link SlotNotFoundException} is thrown.
*
* @param allocationId to identify the task slot to mark as inactive
+ * @param slotTimeout until the slot times out
* @throws SlotNotFoundException if the slot could not be found for the given allocation id
* @return True if the slot could be marked inactive
*/
- public boolean markSlotInactive(AllocationID allocationId) throws SlotNotFoundException {
+ public boolean markSlotInactive(AllocationID allocationId, Time slotTimeout) throws SlotNotFoundException {
checkInit();
TaskSlot taskSlot = getTaskSlot(allocationId);
@@ -253,6 +250,12 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
*/
public int freeSlot(AllocationID allocationId, Throwable cause) throws SlotNotFoundException {
checkInit();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Free slot {}.", allocationId, cause);
+ } else {
+ LOG.info("Free slot {}.", allocationId);
+ }
TaskSlot taskSlot = getTaskSlot(allocationId);
@@ -322,8 +325,6 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
* @return True if the given task slot is allocated for the given job and allocation id
*/
public boolean isAllocated(int index, JobID jobId, AllocationID allocationId) {
- checkInit();
-
TaskSlot taskSlot = taskSlots.get(index);
return taskSlot.isAllocated(jobId, allocationId);
@@ -336,7 +337,7 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
* @param allocationId identifying the allocation
* @return True if there exists a task slot which is active for the given job and allocation id.
*/
- public boolean existActiveSlot(JobID jobId, AllocationID allocationId) {
+ public boolean existsActiveSlot(JobID jobId, AllocationID allocationId) {
TaskSlot taskSlot = getTaskSlot(allocationId);
if (taskSlot != null) {
@@ -431,6 +432,8 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
* @return The removed task if there is any for the given execution attempt id; otherwise null
*/
public Task removeTask(ExecutionAttemptID executionAttemptID) {
+ checkInit();
+
TaskSlotMapping taskSlotMapping = taskSlotMappings.remove(executionAttemptID);
if (taskSlotMapping != null) {
@@ -481,6 +484,8 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
@Override
public void notifyTimeout(AllocationID key, UUID ticket) {
+ checkInit();
+
if (slotActions != null) {
slotActions.timeoutSlot(key, ticket);
}
@@ -493,9 +498,7 @@ public class TaskSlotTable implements TimeoutListener<AllocationID> {
private TaskSlot getTaskSlot(AllocationID allocationId) {
Preconditions.checkNotNull(allocationId);
- TaskSlot taskSlot = allocationIDTaskSlotMap.get(allocationId);
-
- return taskSlot;
+ return allocationIDTaskSlotMap.get(allocationId);
}
private void checkInit() {
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index 096bcef..7e9fbfa 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -273,7 +273,7 @@ public class Task implements Runnable, TaskActions {
this.jobId = checkNotNull(tdd.getJobID());
this.vertexId = checkNotNull(tdd.getVertexID());
this.executionId = checkNotNull(tdd.getExecutionId());
- this.allocationId = checkNotNull(tdd.getAllocationID());
+ this.allocationId = checkNotNull(tdd.getAllocationId());
this.taskNameWithSubtask = taskInfo.getTaskNameWithSubtasks();
this.jobConfiguration = checkNotNull(tdd.getJobConfiguration());
this.taskConfiguration = checkNotNull(tdd.getTaskConfiguration());
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java
index 39ea176..993fd19 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java
@@ -29,6 +29,7 @@ import java.util.List;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.blob.BlobKey;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.jobgraph.JobVertexID;
@@ -44,6 +45,7 @@ public class TaskDeploymentDescriptorTest {
public void testSerialization() {
try {
final JobID jobID = new JobID();
+ final AllocationID allocationId = new AllocationID();
final JobVertexID vertexID = new JobVertexID();
final ExecutionAttemptID execId = new ExecutionAttemptID();
final String jobName = "job name";
@@ -61,7 +63,7 @@ public class TaskDeploymentDescriptorTest {
final List<URL> requiredClasspaths = new ArrayList<URL>(0);
final SerializedValue<ExecutionConfig> executionConfig = new SerializedValue<>(new ExecutionConfig());
- final TaskDeploymentDescriptor orig = new TaskDeploymentDescriptor(jobID, jobName, vertexID, execId,
+ final TaskDeploymentDescriptor orig = new TaskDeploymentDescriptor(jobID, allocationId, jobName, vertexID, execId,
executionConfig, taskName, numberOfKeyGroups, indexInSubtaskGroup, currentNumberOfSubtasks, attemptNumber,
jobConfiguration, taskConfiguration, invokableClass.getName(), producedResults, inputGates,
requiredJars, requiredClasspaths, 47);
@@ -69,12 +71,14 @@ public class TaskDeploymentDescriptorTest {
final TaskDeploymentDescriptor copy = CommonTestUtils.createCopySerializable(orig);
assertFalse(orig.getJobID() == copy.getJobID());
+ assertFalse(orig.getAllocationId() == copy.getAllocationId());
assertFalse(orig.getVertexID() == copy.getVertexID());
assertFalse(orig.getTaskName() == copy.getTaskName());
assertFalse(orig.getJobConfiguration() == copy.getJobConfiguration());
assertFalse(orig.getTaskConfiguration() == copy.getTaskConfiguration());
assertEquals(orig.getJobID(), copy.getJobID());
+ assertEquals(orig.getAllocationId(), copy.getAllocationId());
assertEquals(orig.getVertexID(), copy.getVertexID());
assertEquals(orig.getTaskName(), copy.getTaskName());
assertEquals(orig.getNumberOfKeyGroups(), copy.getNumberOfKeyGroups());
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
index a255027..38e372d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
@@ -91,7 +91,7 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
}
@Override
- public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception {
+ public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) throws Exception {
LeaderRetrievalService service = this.jobMasterLeaderRetrievers.get(jobID);
if (service != null) {
return service;
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskManagerGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskManagerGroupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskManagerGroupTest.java
index 49f2268..073aeac 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskManagerGroupTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskManagerGroupTest.java
@@ -23,6 +23,7 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.blob.BlobKey;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
import org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor;
import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
@@ -75,7 +76,8 @@ public class TaskManagerGroupTest extends TestLogger {
final ExecutionAttemptID execution21 = new ExecutionAttemptID();
TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(
- jid1,
+ jid1,
+ new AllocationID(),
jobName1,
vertex11,
execution11,
@@ -91,6 +93,7 @@ public class TaskManagerGroupTest extends TestLogger {
TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(
jid1,
+ new AllocationID(),
jobName1,
vertex12,
execution12,
@@ -106,6 +109,7 @@ public class TaskManagerGroupTest extends TestLogger {
TaskDeploymentDescriptor tdd3 = new TaskDeploymentDescriptor(
jid2,
+ new AllocationID(),
jobName2,
vertex21,
execution21,
@@ -121,6 +125,7 @@ public class TaskManagerGroupTest extends TestLogger {
TaskDeploymentDescriptor tdd4 = new TaskDeploymentDescriptor(
jid1,
+ new AllocationID(),
jobName1,
vertex13,
execution13,
@@ -192,6 +197,7 @@ public class TaskManagerGroupTest extends TestLogger {
TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(
jid1,
+ new AllocationID(),
jobName1,
vertex11,
execution11,
@@ -207,6 +213,7 @@ public class TaskManagerGroupTest extends TestLogger {
TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(
jid1,
+ new AllocationID(),
jobName1,
vertex12,
execution12,
@@ -222,6 +229,7 @@ public class TaskManagerGroupTest extends TestLogger {
TaskDeploymentDescriptor tdd3 = new TaskDeploymentDescriptor(
jid2,
+ new AllocationID(),
jobName2,
vertex21,
execution21,
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
index 558d3c2..948c129 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
@@ -68,7 +68,7 @@ public class SlotManagerTest {
taskExecutorRegistration = Mockito.mock(TaskExecutorRegistration.class);
TaskExecutorGateway gateway = Mockito.mock(TaskExecutorGateway.class);
Mockito.when(taskExecutorRegistration.getTaskExecutorGateway()).thenReturn(gateway);
- Mockito.when(gateway.requestSlot(any(SlotID.class), any(AllocationID.class), any(UUID.class), any(Time.class)))
+ Mockito.when(gateway.requestSlot(any(SlotID.class), any(JobID.class), any(AllocationID.class), any(String.class), any(UUID.class), any(Time.class)))
.thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
index 24d959e..86cd1f8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
@@ -144,7 +144,7 @@ public class SlotProtocolTest extends TestLogger {
Mockito
.when(
taskExecutorGateway
- .requestSlot(any(SlotID.class), any(AllocationID.class), any(UUID.class), any(Time.class)))
+ .requestSlot(any(SlotID.class), any(JobID.class), any(AllocationID.class), any(String.class), any(UUID.class), any(Time.class)))
.thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
testRpcService.registerGateway(tmAddress, taskExecutorGateway);
@@ -161,7 +161,7 @@ public class SlotProtocolTest extends TestLogger {
// 4) Slot becomes available and TaskExecutor gets a SlotRequest
verify(taskExecutorGateway, timeout(5000))
- .requestSlot(eq(slotID), eq(allocationID), any(UUID.class), any(Time.class));
+ .requestSlot(eq(slotID), eq(jobID), eq(allocationID), any(String.class), any(UUID.class), any(Time.class));
}
/**
@@ -189,7 +189,7 @@ public class SlotProtocolTest extends TestLogger {
TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
Mockito.when(
taskExecutorGateway
- .requestSlot(any(SlotID.class), any(AllocationID.class), any(UUID.class), any(Time.class)))
+ .requestSlot(any(SlotID.class), any(JobID.class), any(AllocationID.class), any(String.class), any(UUID.class), any(Time.class)))
.thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
testRpcService.registerGateway(tmAddress, taskExecutorGateway);
@@ -240,7 +240,7 @@ public class SlotProtocolTest extends TestLogger {
// 4) a SlotRequest is routed to the TaskExecutor
verify(taskExecutorGateway, timeout(5000))
- .requestSlot(eq(slotID), eq(allocationID), any(UUID.class), any(Time.class));
+ .requestSlot(eq(slotID), eq(jobID), eq(allocationID), any(String.class), any(UUID.class), any(Time.class));
}
private static TestingLeaderElectionService configureHA(
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcCompletenessTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcCompletenessTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcCompletenessTest.java
index e7143ae..bbde331 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcCompletenessTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/RpcCompletenessTest.java
@@ -38,6 +38,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -287,17 +288,27 @@ public class RpcCompletenessTest extends TestLogger {
if (!futureClass.equals(RpcCompletenessTest.futureClass)) {
return false;
} else {
- Class<?> valueClass = ReflectionUtil.getTemplateType1(gatewayMethod.getGenericReturnType());
+ ReflectionUtil.FullTypeInfo fullValueTypeInfo = ReflectionUtil.getFullTemplateType(gatewayMethod.getGenericReturnType(), 0);
if (endpointMethod.getReturnType().equals(futureClass)) {
- Class<?> rpcEndpointValueClass = ReflectionUtil.getTemplateType1(endpointMethod.getGenericReturnType());
+ ReflectionUtil.FullTypeInfo fullRpcEndpointValueTypeInfo = ReflectionUtil.getFullTemplateType(endpointMethod.getGenericReturnType(), 0);
// check if we have the same future value types
- if (valueClass != null && rpcEndpointValueClass != null && !checkType(valueClass, rpcEndpointValueClass)) {
- return false;
+ if (fullValueTypeInfo != null && fullRpcEndpointValueTypeInfo != null) {
+ Iterator<Class<?>> valueClasses = fullValueTypeInfo.getClazzIterator();
+ Iterator<Class<?>> rpcClasses = fullRpcEndpointValueTypeInfo.getClazzIterator();
+
+ while (valueClasses.hasNext() && rpcClasses.hasNext()) {
+ if (!checkType(valueClasses.next(), rpcClasses.next())) {
+ return false;
+ }
+ }
+
+ // both should be empty
+ return !valueClasses.hasNext() && !rpcClasses.hasNext();
}
} else {
- if (valueClass != null && !checkType(valueClass, endpointMethod.getReturnType())) {
+ if (fullValueTypeInfo != null && !checkType(fullValueTypeInfo.getClazz(), endpointMethod.getReturnType())) {
return false;
}
}
@@ -342,16 +353,16 @@ public class RpcCompletenessTest extends TestLogger {
if (method.getReturnType().equals(Void.TYPE)) {
builder.append("void").append(" ");
} else if (method.getReturnType().equals(futureClass)) {
- Class<?> valueClass = ReflectionUtil.getTemplateType1(method.getGenericReturnType());
+ ReflectionUtil.FullTypeInfo fullTypeInfo = ReflectionUtil.getFullTemplateType(method.getGenericReturnType(), 0);
builder
.append(futureClass.getSimpleName())
.append("<")
- .append(valueClass != null ? valueClass.getSimpleName() : "")
+ .append(fullTypeInfo != null ? fullTypeInfo.toString() : "")
.append(">");
- if (valueClass != null) {
- builder.append("/").append(valueClass.getSimpleName());
+ if (fullTypeInfo != null) {
+ builder.append("/").append(fullTypeInfo);
}
builder.append(" ");
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index baae251..23c6833 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -18,21 +18,45 @@
package org.apache.flink.runtime.taskexecutor;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.clusterframework.types.SlotID;
+import org.apache.flink.runtime.concurrent.CompletableFuture;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
+import org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor;
+import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
+import org.apache.flink.runtime.execution.librarycache.LibraryCacheManager;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.filecache.FileCache;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.highavailability.NonHaServices;
import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
+import org.apache.flink.runtime.instance.InstanceID;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.network.NetworkEnvironment;
+import org.apache.flink.runtime.io.network.netty.PartitionStateChecker;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionConsumableNotifier;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+import org.apache.flink.runtime.jobmaster.JMTMRegistrationSuccess;
+import org.apache.flink.runtime.jobmaster.JobMasterGateway;
import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.memory.MemoryManager;
import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
+import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
+import org.apache.flink.runtime.query.TaskKvStateRegistry;
+import org.apache.flink.runtime.registration.RegistrationResponse;
import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRegistered;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRejected;
@@ -40,19 +64,42 @@ import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequ
import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.TestingSerialRpcService;
import org.apache.flink.runtime.taskexecutor.slot.TaskSlotTable;
+import org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException;
+import org.apache.flink.runtime.taskexecutor.slot.TimerService;
+import org.apache.flink.runtime.taskmanager.CheckpointResponder;
+import org.apache.flink.runtime.taskmanager.Task;
+import org.apache.flink.runtime.taskmanager.TaskManagerActions;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
+import org.apache.flink.util.SerializedValue;
import org.apache.flink.util.TestLogger;
+import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
+import org.mockito.Matchers;
import org.powermock.api.mockito.PowerMockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
+import static org.hamcrest.Matchers.contains;
public class TaskExecutorTest extends TestLogger {
+ @Rule
+ public TestName name = new TestName();
+
+
@Test
public void testImmediatelyRegistersIfLeaderIsKnown() throws Exception {
final ResourceID resourceID = ResourceID.generate();
@@ -85,6 +132,8 @@ public class TaskExecutorTest extends TestLogger {
mock(BroadcastVariableManager.class),
mock(FileCache.class),
mock(TaskSlotTable.class),
+ mock(JobManagerTable.class),
+ mock(JobLeaderService.class),
mock(FatalErrorHandler.class));
taskManager.start();
@@ -142,6 +191,8 @@ public class TaskExecutorTest extends TestLogger {
mock(BroadcastVariableManager.class),
mock(FileCache.class),
mock(TaskSlotTable.class),
+ mock(JobManagerTable.class),
+ mock(JobLeaderService.class),
mock(FatalErrorHandler.class));
taskManager.start();
@@ -173,17 +224,385 @@ public class TaskExecutorTest extends TestLogger {
}
/**
+ * Tests that we can submit a task to the TaskManager given that we've allocated a slot there.
+ */
+ @Test(timeout = 1000L)
+ public void testTaskSubmission() throws Exception {
+ final Configuration configuration = new Configuration();
+
+ final TestingSerialRpcService rpc = new TestingSerialRpcService();
+ final TaskManagerConfiguration taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration);
+ final JobID jobId = new JobID();
+ final AllocationID allocationId = new AllocationID();
+ final UUID jobManagerLeaderId = UUID.randomUUID();
+ final JobVertexID jobVertexId = new JobVertexID();
+
+ final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
+ jobId,
+ allocationId,
+ name.getMethodName(),
+ jobVertexId,
+ new ExecutionAttemptID(),
+ new SerializedValue<>(new ExecutionConfig()),
+ "test task",
+ 1,
+ 0,
+ 1,
+ 0,
+ configuration,
+ configuration,
+ TestInvokable.class.getName(),
+ Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
+ Collections.<InputGateDeploymentDescriptor>emptyList(),
+ Collections.<BlobKey>emptyList(),
+ Collections.<URL>emptyList(),
+ 0);
+
+ final LibraryCacheManager libraryCacheManager = mock(LibraryCacheManager.class);
+ when(libraryCacheManager.getClassLoader(eq(jobId))).thenReturn(getClass().getClassLoader());
+
+ final JobManagerConnection jobManagerConnection = new JobManagerConnection(
+ mock(JobMasterGateway.class),
+ jobManagerLeaderId,
+ mock(TaskManagerActions.class),
+ mock(CheckpointResponder.class),
+ libraryCacheManager,
+ mock(ResultPartitionConsumableNotifier.class),
+ mock(PartitionStateChecker.class));
+
+ final JobManagerTable jobManagerTable = new JobManagerTable();
+ jobManagerTable.put(jobId, jobManagerConnection);
+
+ final TaskSlotTable taskSlotTable = mock(TaskSlotTable.class);
+ when(taskSlotTable.existsActiveSlot(eq(jobId), eq(allocationId))).thenReturn(true);
+ when(taskSlotTable.addTask(any(Task.class))).thenReturn(true);
+
+ final NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
+
+ when(networkEnvironment.createKvStateTaskRegistry(eq(jobId), eq(jobVertexId))).thenReturn(mock(TaskKvStateRegistry.class));
+
+ final TaskManagerMetricGroup taskManagerMetricGroup = mock(TaskManagerMetricGroup.class);
+
+ when(taskManagerMetricGroup.addTaskForJob(eq(tdd))).thenReturn(mock(TaskMetricGroup.class));
+
+ final HighAvailabilityServices haServices = mock(HighAvailabilityServices.class);
+ when(haServices.getResourceManagerLeaderRetriever()).thenReturn(mock(LeaderRetrievalService.class));
+
+ try {
+
+ TaskExecutor taskManager = new TaskExecutor(
+ taskManagerConfiguration,
+ mock(TaskManagerLocation.class),
+ rpc,
+ mock(MemoryManager.class),
+ mock(IOManager.class),
+ networkEnvironment,
+ haServices,
+ mock(MetricRegistry.class),
+ taskManagerMetricGroup,
+ mock(BroadcastVariableManager.class),
+ mock(FileCache.class),
+ taskSlotTable,
+ jobManagerTable,
+ mock(JobLeaderService.class),
+ mock(FatalErrorHandler.class));
+
+ taskManager.start();
+
+ taskManager.submitTask(tdd, jobManagerLeaderId);
+
+ Future<Boolean> completionFuture = TestInvokable.completableFuture;
+
+ completionFuture.get();
+
+ } finally {
+ rpc.stopService();
+ }
+ }
+
+ /**
+ * Test invokable which completes the given future when executed.
+ */
+ public static class TestInvokable extends AbstractInvokable {
+
+ static final CompletableFuture<Boolean> completableFuture = new FlinkCompletableFuture<>();
+
+ @Override
+ public void invoke() throws Exception {
+ completableFuture.complete(true);
+ }
+ }
+
+ /**
+ * Tests that a TaskManager detects a job leader for which has reserved slots. Upon detecting
+ * the job leader, it will offer all reserved slots to the JobManager.
+ */
+ @Test
+ public void testJobLeaderDetection() throws TestingFatalErrorHandler.TestingException, SlotAllocationException {
+ final JobID jobId = new JobID();
+
+ final TestingSerialRpcService rpc = new TestingSerialRpcService();
+ final Configuration configuration = new Configuration();
+ final TaskManagerConfiguration taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration);
+ final ResourceID resourceId = new ResourceID("foobar");
+ final TaskManagerLocation taskManagerLocation = new TaskManagerLocation(resourceId, InetAddress.getLoopbackAddress(), 1234);
+ final TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServices();
+ final TimerService<AllocationID> timerService = mock(TimerService.class);
+ final TaskSlotTable taskSlotTable = new TaskSlotTable(Arrays.asList(mock(ResourceProfile.class)), timerService);
+ final JobManagerTable jobManagerTable = new JobManagerTable();
+ final JobLeaderService jobLeaderService = new JobLeaderService(resourceId);
+ final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
+
+ final TestingLeaderRetrievalService resourceManagerLeaderRetrievalService = new TestingLeaderRetrievalService();
+ final TestingLeaderRetrievalService jobManagerLeaderRetrievalService = new TestingLeaderRetrievalService();
+ haServices.setResourceManagerLeaderRetriever(resourceManagerLeaderRetrievalService);
+ haServices.setJobMasterLeaderRetriever(jobId, jobManagerLeaderRetrievalService);
+
+ final String resourceManagerAddress = "rm";
+ final UUID resourceManagerLeaderId = UUID.randomUUID();
+
+ final ResourceManagerGateway resourceManagerGateway = mock(ResourceManagerGateway.class);
+ final InstanceID registrationId = new InstanceID();
+
+ when(resourceManagerGateway.registerTaskExecutor(
+ eq(resourceManagerLeaderId),
+ any(String.class),
+ eq(resourceId),
+ any(SlotReport.class),
+ any(Time.class))).thenReturn(FlinkCompletableFuture.<RegistrationResponse>completed(new TaskExecutorRegistrationSuccess(registrationId, 1000L)));
+
+ final String jobManagerAddress = "jm";
+ final UUID jobManagerLeaderId = UUID.randomUUID();
+ final ResourceID jmResourceId = new ResourceID(jobManagerAddress);
+ final int blobPort = 42;
+
+ final JobMasterGateway jobMasterGateway = mock(JobMasterGateway.class);
+
+ when(jobMasterGateway.registerTaskManager(
+ any(String.class),
+ eq(resourceId),
+ eq(jobManagerLeaderId),
+ any(Time.class)
+ )).thenReturn(FlinkCompletableFuture.<RegistrationResponse>completed(new JMTMRegistrationSuccess(jmResourceId, blobPort)));
+ when(jobMasterGateway.getAddress()).thenReturn(jobManagerAddress);
+
+ rpc.registerGateway(resourceManagerAddress, resourceManagerGateway);
+ rpc.registerGateway(jobManagerAddress, jobMasterGateway);
+
+ final AllocationID allocationId = new AllocationID();
+ final SlotID slotId = new SlotID(resourceId, 0);
+
+ try {
+ TaskExecutor taskManager = new TaskExecutor(
+ taskManagerConfiguration,
+ taskManagerLocation,
+ rpc,
+ mock(MemoryManager.class),
+ mock(IOManager.class),
+ mock(NetworkEnvironment.class),
+ haServices,
+ mock(MetricRegistry.class),
+ mock(TaskManagerMetricGroup.class),
+ mock(BroadcastVariableManager.class),
+ mock(FileCache.class),
+ taskSlotTable,
+ jobManagerTable,
+ jobLeaderService,
+ testingFatalErrorHandler);
+
+ taskManager.start();
+
+ // tell the task manager about the rm leader
+ resourceManagerLeaderRetrievalService.notifyListener(resourceManagerAddress, resourceManagerLeaderId);
+
+ // request slots from the task manager under the given allocation id
+ TMSlotRequestReply reply = taskManager.requestSlot(slotId, jobId, allocationId, jobManagerAddress, resourceManagerLeaderId);
+
+ // this is hopefully successful :-)
+ assertTrue(reply instanceof TMSlotRequestRegistered);
+
+ // now inform the task manager about the new job leader
+ jobManagerLeaderRetrievalService.notifyListener(jobManagerAddress, jobManagerLeaderId);
+
+ // the job leader should get the allocation id offered
+ verify(jobMasterGateway).offerSlots((Iterable<AllocationID>)Matchers.argThat(contains(allocationId)), eq(jobManagerLeaderId), any(Time.class));
+ } finally {
+ // check if a concurrent error occurred
+ testingFatalErrorHandler.rethrowException();
+
+ rpc.stopService();
+ }
+ }
+
+ /**
+ * Tests that accepted slots go into state assigned and the others are returned to the resource
+ * manager.
+ */
+ @Test
+ public void testSlotAcceptance() throws Exception {
+ final JobID jobId = new JobID();
+
+ final TestingSerialRpcService rpc = new TestingSerialRpcService();
+ final Configuration configuration = new Configuration();
+ final TaskManagerConfiguration taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration);
+ final ResourceID resourceId = new ResourceID("foobar");
+ final TaskManagerLocation taskManagerLocation = new TaskManagerLocation(resourceId, InetAddress.getLoopbackAddress(), 1234);
+ final TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServices();
+ final TimerService<AllocationID> timerService = mock(TimerService.class);
+ final TaskSlotTable taskSlotTable = new TaskSlotTable(Arrays.asList(mock(ResourceProfile.class), mock(ResourceProfile.class)), timerService);
+ final JobManagerTable jobManagerTable = new JobManagerTable();
+ final JobLeaderService jobLeaderService = new JobLeaderService(resourceId);
+ final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
+
+ final String resourceManagerAddress = "rm";
+ final UUID resourceManagerLeaderId = UUID.randomUUID();
+
+ final String jobManagerAddress = "jm";
+ final UUID jobManagerLeaderId = UUID.randomUUID();
+
+ final LeaderRetrievalService resourceManagerLeaderRetrievalService = new TestingLeaderRetrievalService(resourceManagerAddress, resourceManagerLeaderId);
+ final LeaderRetrievalService jobManagerLeaderRetrievalService = new TestingLeaderRetrievalService(jobManagerAddress, jobManagerLeaderId);
+ haServices.setResourceManagerLeaderRetriever(resourceManagerLeaderRetrievalService);
+ haServices.setJobMasterLeaderRetriever(jobId, jobManagerLeaderRetrievalService);
+
+ final ResourceManagerGateway resourceManagerGateway = mock(ResourceManagerGateway.class);
+ final InstanceID registrationId = new InstanceID();
+
+ when(resourceManagerGateway.registerTaskExecutor(
+ eq(resourceManagerLeaderId),
+ any(String.class),
+ eq(resourceId),
+ any(SlotReport.class),
+ any(Time.class))).thenReturn(FlinkCompletableFuture.<RegistrationResponse>completed(new TaskExecutorRegistrationSuccess(registrationId, 1000L)));
+
+ final ResourceID jmResourceId = new ResourceID(jobManagerAddress);
+ final int blobPort = 42;
+
+ final AllocationID allocationId1 = new AllocationID();
+ final AllocationID allocationId2 = new AllocationID();
+
+ final JobMasterGateway jobMasterGateway = mock(JobMasterGateway.class);
+
+ when(jobMasterGateway.registerTaskManager(
+ any(String.class),
+ eq(resourceId),
+ eq(jobManagerLeaderId),
+ any(Time.class)
+ )).thenReturn(FlinkCompletableFuture.<RegistrationResponse>completed(new JMTMRegistrationSuccess(jmResourceId, blobPort)));
+ when(jobMasterGateway.getAddress()).thenReturn(jobManagerAddress);
+
+ when(jobMasterGateway.offerSlots(any(Iterable.class), eq(jobManagerLeaderId), any(Time.class)))
+ .thenReturn(FlinkCompletableFuture.completed((Iterable<AllocationID>)Collections.singleton(allocationId1)));
+
+ rpc.registerGateway(resourceManagerAddress, resourceManagerGateway);
+ rpc.registerGateway(jobManagerAddress, jobMasterGateway);
+
+ try {
+ TaskExecutor taskManager = new TaskExecutor(
+ taskManagerConfiguration,
+ taskManagerLocation,
+ rpc,
+ mock(MemoryManager.class),
+ mock(IOManager.class),
+ mock(NetworkEnvironment.class),
+ haServices,
+ mock(MetricRegistry.class),
+ mock(TaskManagerMetricGroup.class),
+ mock(BroadcastVariableManager.class),
+ mock(FileCache.class),
+ taskSlotTable,
+ jobManagerTable,
+ jobLeaderService,
+ testingFatalErrorHandler);
+
+ taskManager.start();
+
+ taskSlotTable.allocateSlot(0, jobId, allocationId1, Time.milliseconds(10000L));
+ taskSlotTable.allocateSlot(1, jobId, allocationId2, Time.milliseconds(10000L));
+
+ // we have to add the job after the TaskExecutor, because otherwise the service has not
+ // been properly started.
+ jobLeaderService.addJob(jobId, jobManagerAddress);
+
+ verify(resourceManagerGateway).notifySlotAvailable(eq(resourceManagerLeaderId), eq(registrationId), eq(new SlotID(resourceId, 1)));
+
+ assertTrue(taskSlotTable.existsActiveSlot(jobId, allocationId1));
+ assertFalse(taskSlotTable.existsActiveSlot(jobId, allocationId2));
+ assertTrue(taskSlotTable.isSlotFree(1));
+ } finally {
+ // check if a concurrent error occurred
+ testingFatalErrorHandler.rethrowException();
+
+ rpc.stopService();
+ }
+ }
+
+ private static class TestingFatalErrorHandler implements FatalErrorHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(TestingFatalErrorHandler.class);
+ private final AtomicReference<Throwable> atomicThrowable;
+
+ public TestingFatalErrorHandler() {
+ atomicThrowable = new AtomicReference<>(null);
+ }
+
+ public void rethrowException() throws TestingException {
+ Throwable throwable = atomicThrowable.get();
+
+ if (throwable != null) {
+ throw new TestingException(throwable);
+ }
+ }
+
+ public boolean hasExceptionOccurred() {
+ return atomicThrowable.get() != null;
+ }
+
+ public Throwable getException() {
+ return atomicThrowable.get();
+ }
+
+ @Override
+ public void onFatalError(Throwable exception) {
+ LOG.error("OnFatalError:", exception);
+ atomicThrowable.compareAndSet(null, exception);
+ }
+
+ //------------------------------------------------------------------
+ // static utility classes
+ //------------------------------------------------------------------
+
+ private static final class TestingException extends Exception {
+ public TestingException(String message) {
+ super(message);
+ }
+
+ public TestingException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public TestingException(Throwable cause) {
+ super(cause);
+ }
+
+ private static final long serialVersionUID = -4648195335470914498L;
+ }
+ }
+
+ /**
* Tests that all allocation requests for slots are ignored if the slot has been reported as
* free by the TaskExecutor but this report hasn't been confirmed by the ResourceManager.
*
* This is essential for the correctness of the state of the ResourceManager.
*/
+ @Ignore
@Test
- public void testRejectAllocationRequestsForOutOfSyncSlots() {
+ public void testRejectAllocationRequestsForOutOfSyncSlots() throws SlotAllocationException {
final ResourceID resourceID = ResourceID.generate();
final String address1 = "/resource/manager/address/one";
final UUID leaderId = UUID.randomUUID();
+ final JobID jobId = new JobID();
+ final String jobManagerAddress = "foobar";
final TestingSerialRpcService rpc = new TestingSerialRpcService();
try {
@@ -215,6 +634,8 @@ public class TaskExecutorTest extends TestLogger {
mock(BroadcastVariableManager.class),
mock(FileCache.class),
mock(TaskSlotTable.class),
+ mock(JobManagerTable.class),
+ mock(JobLeaderService.class),
mock(FatalErrorHandler.class));
taskManager.start();
@@ -232,14 +653,14 @@ public class TaskExecutorTest extends TestLogger {
// test that allocating a slot works
final SlotID slotID = new SlotID(resourceID, 0);
- TMSlotRequestReply tmSlotRequestReply = taskManager.requestSlot(slotID, new AllocationID(), leaderId);
+ TMSlotRequestReply tmSlotRequestReply = taskManager.requestSlot(slotID, jobId, new AllocationID(), jobManagerAddress, leaderId);
assertTrue(tmSlotRequestReply instanceof TMSlotRequestRegistered);
+ // TODO: Figure out the concrete allocation behaviour between RM and TM. Maybe we don't need the SlotID...
// test that we can't allocate slots which are blacklisted due to pending confirmation of the RM
final SlotID unconfirmedFreeSlotID = new SlotID(resourceID, 1);
- taskManager.addUnconfirmedFreeSlotNotification(unconfirmedFreeSlotID);
TMSlotRequestReply tmSlotRequestReply2 =
- taskManager.requestSlot(unconfirmedFreeSlotID, new AllocationID(), leaderId);
+ taskManager.requestSlot(unconfirmedFreeSlotID, jobId, new AllocationID(), jobManagerAddress, leaderId);
assertTrue(tmSlotRequestReply2 instanceof TMSlotRequestRejected);
// re-register
@@ -250,7 +671,7 @@ public class TaskExecutorTest extends TestLogger {
// now we should be successful because the slots status has been synced
// test that we can't allocate slots which are blacklisted due to pending confirmation of the RM
TMSlotRequestReply tmSlotRequestReply3 =
- taskManager.requestSlot(unconfirmedFreeSlotID, new AllocationID(), leaderId);
+ taskManager.requestSlot(unconfirmedFreeSlotID, jobId, new AllocationID(), jobManagerAddress, leaderId);
assertTrue(tmSlotRequestReply3 instanceof TMSlotRequestRegistered);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
index fefc00b80..7494d7a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
@@ -22,6 +22,7 @@ import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.testutils.OneShotLatch;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.checkpoint.CheckpointMetaData;
@@ -154,7 +155,7 @@ public class TaskAsyncCallTest {
.thenReturn(mock(TaskKvStateRegistry.class));
TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
- new JobID(), "Job Name", new JobVertexID(), new ExecutionAttemptID(),
+ new JobID(), new AllocationID(), "Job Name", new JobVertexID(), new ExecutionAttemptID(),
new SerializedValue<>(new ExecutionConfig()),
"Test Task", 1, 0, 1, 0,
new Configuration(), new Configuration(),
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
index a203130..b8893fc 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
@@ -33,6 +33,7 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.akka.FlinkUntypedActor;
import org.apache.flink.runtime.blob.BlobKey;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.concurrent.CompletableFuture;
import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
import org.apache.flink.runtime.deployment.InputChannelDeploymentDescriptor;
@@ -170,12 +171,13 @@ public class TaskManagerTest extends TestLogger {
final ExecutionAttemptID eid = new ExecutionAttemptID();
final SerializedValue<ExecutionConfig> executionConfig = new SerializedValue<>(new ExecutionConfig());
- final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(jid, "TestJob", vid, eid, executionConfig,
- "TestTask", 7, 2, 7, 0, new Configuration(), new Configuration(),
- TestInvokableCorrect.class.getName(),
- Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
- Collections.<InputGateDeploymentDescriptor>emptyList(),
- new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
+ final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(jid, new AllocationID(),
+ "TestJob", vid, eid, executionConfig,
+ "TestTask", 7, 2, 7, 0, new Configuration(), new Configuration(),
+ TestInvokableCorrect.class.getName(),
+ Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
+ Collections.<InputGateDeploymentDescriptor>emptyList(),
+ new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
new Within(d) {
@@ -268,7 +270,7 @@ public class TaskManagerTest extends TestLogger {
final ExecutionAttemptID eid2 = new ExecutionAttemptID();
final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(
- jid1, "TestJob1", vid1, eid1,
+ jid1, new AllocationID(), "TestJob1", vid1, eid1,
new SerializedValue<>(new ExecutionConfig()),
"TestTask1", 5, 1, 5, 0,
new Configuration(), new Configuration(), TestInvokableBlockingCancelable.class.getName(),
@@ -277,7 +279,7 @@ public class TaskManagerTest extends TestLogger {
new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(
- jid2, "TestJob2", vid2, eid2,
+ jid2, new AllocationID(), "TestJob2", vid2, eid2,
new SerializedValue<>(new ExecutionConfig()),
"TestTask2", 7, 2, 7, 0,
new Configuration(), new Configuration(), TestInvokableBlockingCancelable.class.getName(),
@@ -406,13 +408,13 @@ public class TaskManagerTest extends TestLogger {
final SerializedValue<ExecutionConfig> executionConfig = new SerializedValue<>(new ExecutionConfig());
- final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid1, "TestJob", vid1, eid1, executionConfig,
+ final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid1, new AllocationID(), "TestJob", vid1, eid1, executionConfig,
"TestTask1", 5, 1, 5, 0, new Configuration(), new Configuration(), StoppableInvokable.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
Collections.<InputGateDeploymentDescriptor>emptyList(),
new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
- final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid2, "TestJob", vid2, eid2, executionConfig,
+ final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid2, new AllocationID(), "TestJob", vid2, eid2, executionConfig,
"TestTask2", 7, 2, 7, 0, new Configuration(), new Configuration(), TestInvokableBlockingCancelable.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
Collections.<InputGateDeploymentDescriptor>emptyList(),
@@ -532,7 +534,7 @@ public class TaskManagerTest extends TestLogger {
final ExecutionAttemptID eid2 = new ExecutionAttemptID();
final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(
- jid, "TestJob", vid1, eid1,
+ jid, new AllocationID(), "TestJob", vid1, eid1,
new SerializedValue<>(new ExecutionConfig()),
"Sender", 1, 0, 1, 0,
new Configuration(), new Configuration(), Tasks.Sender.class.getName(),
@@ -541,7 +543,7 @@ public class TaskManagerTest extends TestLogger {
new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(
- jid, "TestJob", vid2, eid2,
+ jid, new AllocationID(), "TestJob", vid2, eid2,
new SerializedValue<>(new ExecutionConfig()),
"Receiver", 7, 2, 7, 0,
new Configuration(), new Configuration(), Tasks.Receiver.class.getName(),
@@ -637,7 +639,7 @@ public class TaskManagerTest extends TestLogger {
);
final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(
- jid, "TestJob", vid1, eid1,
+ jid, new AllocationID(), "TestJob", vid1, eid1,
new SerializedValue<>(new ExecutionConfig()),
"Sender", 1, 0, 1, 0,
new Configuration(), new Configuration(), Tasks.Sender.class.getName(),
@@ -645,7 +647,7 @@ public class TaskManagerTest extends TestLogger {
Collections.<URL>emptyList(), 0);
final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(
- jid, "TestJob", vid2, eid2,
+ jid, new AllocationID(), "TestJob", vid2, eid2,
new SerializedValue<>(new ExecutionConfig()),
"Receiver", 7, 2, 7, 0,
new Configuration(), new Configuration(), Tasks.Receiver.class.getName(),
@@ -782,7 +784,7 @@ public class TaskManagerTest extends TestLogger {
);
final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(
- jid, "TestJob", vid1, eid1,
+ jid, new AllocationID(), "TestJob", vid1, eid1,
new SerializedValue<>(new ExecutionConfig()),
"Sender", 1, 0, 1, 0,
new Configuration(), new Configuration(), Tasks.Sender.class.getName(),
@@ -790,7 +792,7 @@ public class TaskManagerTest extends TestLogger {
new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(
- jid, "TestJob", vid2, eid2,
+ jid, new AllocationID(), "TestJob", vid2, eid2,
new SerializedValue<>(new ExecutionConfig()),
"Receiver", 7, 2, 7, 0,
new Configuration(), new Configuration(), Tasks.BlockingReceiver.class.getName(),
@@ -930,7 +932,7 @@ public class TaskManagerTest extends TestLogger {
new InputGateDeploymentDescriptor(resultId, 0, icdd);
final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
- jid, "TestJob", vid, eid,
+ jid, new AllocationID(), "TestJob", vid, eid,
new SerializedValue<>(new ExecutionConfig()),
"Receiver", 1, 0, 1, 0,
new Configuration(), new Configuration(),
@@ -1023,7 +1025,7 @@ public class TaskManagerTest extends TestLogger {
new InputGateDeploymentDescriptor(resultId, 0, icdd);
final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
- jid, "TestJob", vid, eid,
+ jid, new AllocationID(), "TestJob", vid, eid,
new SerializedValue<>(new ExecutionConfig()),
"Receiver", 1, 0, 1, 0,
new Configuration(), new Configuration(),
@@ -1098,24 +1100,25 @@ public class TaskManagerTest extends TestLogger {
// Single blocking task
final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
- new JobID(),
- "Job",
- new JobVertexID(),
- new ExecutionAttemptID(),
- new SerializedValue<>(new ExecutionConfig()),
- "Task",
- 1,
- 0,
- 1,
- 0,
- new Configuration(),
- new Configuration(),
- Tasks.BlockingNoOpInvokable.class.getName(),
- Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
- Collections.<InputGateDeploymentDescriptor>emptyList(),
- Collections.<BlobKey>emptyList(),
- Collections.<URL>emptyList(),
- 0);
+ new JobID(),
+ new AllocationID(),
+ "Job",
+ new JobVertexID(),
+ new ExecutionAttemptID(),
+ new SerializedValue<>(new ExecutionConfig()),
+ "Task",
+ 1,
+ 0,
+ 1,
+ 0,
+ new Configuration(),
+ new Configuration(),
+ Tasks.BlockingNoOpInvokable.class.getName(),
+ Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
+ Collections.<InputGateDeploymentDescriptor>emptyList(),
+ Collections.<BlobKey>emptyList(),
+ Collections.<URL>emptyList(),
+ 0);
// Submit the task
new Within(d) {
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
index a2bfad5..fa03706 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
@@ -69,7 +69,7 @@ public class TaskStopTest {
when(tddMock.getTaskConfiguration()).thenReturn(mock(Configuration.class));
when(tddMock.getSerializedExecutionConfig()).thenReturn(mock(SerializedValue.class));
when(tddMock.getInvokableClassName()).thenReturn("className");
- when(tddMock.getAllocationID()).thenReturn(mock(AllocationID.class));
+ when(tddMock.getAllocationId()).thenReturn(mock(AllocationID.class));
task = new Task(
tddMock,
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
index 1d2f9ed..75c1a47 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
@@ -25,6 +25,7 @@ import org.apache.flink.configuration.TaskManagerOptions;
import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
import org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor;
import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
@@ -823,16 +824,18 @@ public class TaskTest extends TestLogger {
}
return new TaskDeploymentDescriptor(
- new JobID(), "Test Job", new JobVertexID(), new ExecutionAttemptID(),
- serializedExecConfig,
- "Test Task", 1, 0, 1, 0,
+ new JobID(),
+ new AllocationID(),
+ "Test Job", new JobVertexID(), new ExecutionAttemptID(),
+ serializedExecConfig,
+ "Test Task", 1, 0, 1, 0,
new Configuration(), taskConfig,
- invokable.getName(),
- Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
- Collections.<InputGateDeploymentDescriptor>emptyList(),
- Collections.<BlobKey>emptyList(),
- Collections.<URL>emptyList(),
- 0);
+ invokable.getName(),
+ Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
+ Collections.<InputGateDeploymentDescriptor>emptyList(),
+ Collections.<BlobKey>emptyList(),
+ Collections.<URL>emptyList(),
+ 0);
}
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java
index 1077052..f095cf4 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java
@@ -25,6 +25,7 @@ import org.apache.flink.core.fs.FSDataInputStream;
import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
import org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor;
import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
@@ -139,21 +140,22 @@ public class InterruptSensitiveRestoreTest {
keyGroupStateFromStream);
return new TaskDeploymentDescriptor(
- new JobID(),
- "test job name",
- new JobVertexID(),
- new ExecutionAttemptID(),
- new SerializedValue<>(new ExecutionConfig()),
- "test task name",
- 1, 0, 1, 0,
- new Configuration(),
- taskConfig,
- SourceStreamTask.class.getName(),
- Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
- Collections.<InputGateDeploymentDescriptor>emptyList(),
- Collections.<BlobKey>emptyList(),
- Collections.<URL>emptyList(),
- 0,
+ new JobID(),
+ new AllocationID(),
+ "test job name",
+ new JobVertexID(),
+ new ExecutionAttemptID(),
+ new SerializedValue<>(new ExecutionConfig()),
+ "test task name",
+ 1, 0, 1, 0,
+ new Configuration(),
+ taskConfig,
+ SourceStreamTask.class.getName(),
+ Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
+ Collections.<InputGateDeploymentDescriptor>emptyList(),
+ Collections.<BlobKey>emptyList(),
+ Collections.<URL>emptyList(),
+ 0,
taskStateHandles);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2b234521/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
index bb246f9..52332a2 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
@@ -27,6 +27,7 @@ import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
import org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor;
import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
@@ -220,17 +221,19 @@ public class StreamTaskTest {
.thenReturn(mock(TaskKvStateRegistry.class));
TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
- new JobID(), "Job Name", new JobVertexID(), new ExecutionAttemptID(),
- new SerializedValue<>(new ExecutionConfig()),
- "Test Task", 1, 0, 1, 0,
- new Configuration(),
- taskConfig.getConfiguration(),
- invokable.getName(),
- Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
- Collections.<InputGateDeploymentDescriptor>emptyList(),
- Collections.<BlobKey>emptyList(),
- Collections.<URL>emptyList(),
- 0);
+ new JobID(),
+ new AllocationID(),
+ "Job Name", new JobVertexID(), new ExecutionAttemptID(),
+ new SerializedValue<>(new ExecutionConfig()),
+ "Test Task", 1, 0, 1, 0,
+ new Configuration(),
+ taskConfig.getConfiguration(),
+ invokable.getName(),
+ Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
+ Collections.<InputGateDeploymentDescriptor>emptyList(),
+ Collections.<BlobKey>emptyList(),
+ Collections.<URL>emptyList(),
+ 0);
return new Task(
tdd,
[10/50] [abbrv] flink git commit: [FLINK-4606] Integrate the new
ResourceManager with the existed FlinkResourceManager
Posted by tr...@apache.org.
[FLINK-4606] Integrate the new ResourceManager with the existed FlinkResourceManager
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/359ee0ad
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/359ee0ad
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/359ee0ad
Branch: refs/heads/flip-6
Commit: 359ee0adf899bee91a1e55a2571601f6adbb5337
Parents: 5ca94fb
Author: beyond1920 <be...@126.com>
Authored: Fri Sep 9 09:11:24 2016 +0800
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:28 2016 +0100
----------------------------------------------------------------------
.../InfoMessageListenerRpcGateway.java | 35 +++
.../resourcemanager/ResourceManager.java | 214 ++++++++++++++++---
.../resourcemanager/ResourceManagerGateway.java | 23 ++
.../StandaloneResourceManager.java | 64 ++++++
.../resourcemanager/ResourceManagerHATest.java | 2 +-
.../ResourceManagerJobMasterTest.java | 2 +-
.../ResourceManagerTaskExecutorTest.java | 2 +-
.../slotmanager/SlotProtocolTest.java | 5 +-
8 files changed, 318 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/359ee0ad/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/InfoMessageListenerRpcGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/InfoMessageListenerRpcGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/InfoMessageListenerRpcGateway.java
new file mode 100644
index 0000000..c1eeefa
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/InfoMessageListenerRpcGateway.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager;
+
+import org.apache.flink.runtime.clusterframework.messages.InfoMessage;
+import org.apache.flink.runtime.jobmaster.JobMaster;
+import org.apache.flink.runtime.rpc.RpcGateway;
+
+/**
+ * A gateway to listen for info messages from {@link ResourceManager}
+ */
+public interface InfoMessageListenerRpcGateway extends RpcGateway {
+
+ /**
+ * Notifies when resource manager need to notify listener about InfoMessage
+ * @param infoMessage
+ */
+ void notifyInfoMessage(InfoMessage infoMessage);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/359ee0ad/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index 88b8a11..83dc4db 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -20,19 +20,22 @@ package org.apache.flink.runtime.resourcemanager;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.clusterframework.ApplicationStatus;
+import org.apache.flink.runtime.clusterframework.messages.InfoMessage;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.AcceptFunction;
+import org.apache.flink.runtime.concurrent.ApplyFunction;
import org.apache.flink.runtime.concurrent.BiFunction;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.instance.InstanceID;
import org.apache.flink.runtime.jobmaster.JobMasterRegistrationSuccess;
import org.apache.flink.runtime.leaderelection.LeaderContender;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
-import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcEndpoint;
+import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.jobmaster.JobMaster;
import org.apache.flink.runtime.jobmaster.JobMasterGateway;
@@ -42,8 +45,6 @@ import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
import org.apache.flink.runtime.taskexecutor.TaskExecutorRegistrationSuccess;
import org.apache.flink.runtime.util.LeaderConnectionInfo;
import org.apache.flink.runtime.util.LeaderRetrievalUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import scala.concurrent.duration.FiniteDuration;
import java.util.HashMap;
@@ -66,15 +67,16 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
* <li>{@link #requestSlot(SlotRequest)} requests a slot from the resource manager</li>
* </ul>
*/
-public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> implements LeaderContender {
+public abstract class ResourceManager<ResourceManagerGateway, WorkerType extends TaskExecutorRegistration> extends RpcEndpoint implements LeaderContender {
- private final Logger LOG = LoggerFactory.getLogger(getClass());
+ /** The exit code with which the process is stopped in case of a fatal error */
+ protected static final int EXIT_CODE_FATAL_ERROR = -13;
private final Map<JobID, JobMasterGateway> jobMasterGateways;
private final Set<LeaderRetrievalListener> jobMasterLeaderRetrievalListeners;
- private final Map<ResourceID, TaskExecutorRegistration> taskExecutorGateways;
+ private final Map<ResourceID, WorkerType> taskExecutorGateways;
private final HighAvailabilityServices highAvailabilityServices;
@@ -84,16 +86,16 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> impleme
private UUID leaderSessionID;
- public ResourceManager(
- RpcService rpcService,
- HighAvailabilityServices highAvailabilityServices,
- SlotManager slotManager) {
+ private Map<String, InfoMessageListenerRpcGateway> infoMessageListeners;
+
+ public ResourceManager(RpcService rpcService, HighAvailabilityServices highAvailabilityServices, SlotManager slotManager) {
super(rpcService);
this.highAvailabilityServices = checkNotNull(highAvailabilityServices);
this.jobMasterGateways = new HashMap<>();
- this.slotManager = slotManager;
+ this.slotManager = checkNotNull(slotManager);
this.jobMasterLeaderRetrievalListeners = new HashSet<>();
this.taskExecutorGateways = new HashMap<>();
+ infoMessageListeners = new HashMap<>();
}
@Override
@@ -103,6 +105,8 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> impleme
super.start();
leaderElectionService = highAvailabilityServices.getResourceManagerLeaderElectionService();
leaderElectionService.start(this);
+ // framework specific initialization
+ initialize();
} catch (Throwable e) {
log.error("A fatal error happened when starting the ResourceManager", e);
throw new RuntimeException("A fatal error happened when starting the ResourceManager", e);
@@ -166,12 +170,12 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> impleme
jobMasterLeaderInfo = LeaderRetrievalUtils.retrieveLeaderConnectionInfo(
highAvailabilityServices.getJobMasterLeaderRetriever(jobID), new FiniteDuration(5, TimeUnit.SECONDS));
} catch (Exception e) {
- LOG.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
+ log.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
throw new Exception("Failed to retrieve JobMasterLeaderRetriever");
}
if (!jobMasterLeaderId.equals(jobMasterLeaderInfo.getLeaderSessionID())) {
- LOG.info("Declining registration request from non-leading JobManager {}", jobMasterAddress);
+ log.info("Declining registration request from non-leading JobManager {}", jobMasterAddress);
throw new Exception("JobManager is not leading");
}
@@ -190,7 +194,7 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> impleme
LeaderRetrievalService jobMasterLeaderRetriever = highAvailabilityServices.getJobMasterLeaderRetriever(jobID);
jobMasterLeaderRetriever.start(jobMasterLeaderListener);
} catch (Exception e) {
- LOG.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
+ log.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
return new RegistrationResponse.Decline("Failed to retrieve JobMasterLeaderRetriever");
}
jobMasterLeaderRetrievalListeners.add(jobMasterLeaderListener);
@@ -237,13 +241,24 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> impleme
if (throwable != null) {
return new RegistrationResponse.Decline(throwable.getMessage());
} else {
- InstanceID id = new InstanceID();
- TaskExecutorRegistration oldTaskExecutor =
- taskExecutorGateways.put(resourceID, new TaskExecutorRegistration(taskExecutorGateway, id));
- if (oldTaskExecutor != null) {
- log.warn("Receive a duplicate registration from TaskExecutor {} at ({})", resourceID, taskExecutorAddress);
+ WorkerType startedWorker = taskExecutorGateways.get(resourceID);
+ if(startedWorker != null) {
+ String oldWorkerAddress = startedWorker.getTaskExecutorGateway().getAddress();
+ if (taskExecutorAddress.equals(oldWorkerAddress)) {
+ log.warn("Receive a duplicate registration from TaskExecutor {} at ({})", resourceID, taskExecutorAddress);
+ } else {
+ log.warn("Receive a duplicate registration from TaskExecutor {} at different address, previous ({}), new ({})",
+ resourceID, oldWorkerAddress, taskExecutorAddress);
+ // TODO :: suggest old taskExecutor to stop itself
+ slotManager.notifyTaskManagerFailure(resourceID);
+ startedWorker = workerStarted(resourceID, taskExecutorGateway);
+ taskExecutorGateways.put(resourceID, startedWorker);
+ }
+ } else {
+ startedWorker = workerStarted(resourceID, taskExecutorGateway);
+ taskExecutorGateways.put(resourceID, startedWorker);
}
- return new TaskExecutorRegistrationSuccess(id, 5000);
+ return new TaskExecutorRegistrationSuccess(startedWorker.getInstanceID(), 5000);
}
}
}, getMainThreadExecutor());
@@ -263,14 +278,12 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> impleme
if (jobMasterGateway != null) {
return slotManager.requestSlot(slotRequest);
} else {
- LOG.info("Ignoring slot request for unknown JobMaster with JobID {}", jobId);
+ log.info("Ignoring slot request for unknown JobMaster with JobID {}", jobId);
return new SlotRequestRejected(slotRequest.getAllocationId());
}
}
-
-
// ------------------------------------------------------------------------
// Leader Contender
// ------------------------------------------------------------------------
@@ -324,6 +337,158 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> impleme
shutDown();
}
+ /**
+ * Registers an infoMessage listener
+ *
+ * @param infoMessageListenerAddress address of infoMessage listener to register to this resource manager
+ */
+ @RpcMethod
+ public void registerInfoMessageListener(final String infoMessageListenerAddress) {
+ if(infoMessageListeners.containsKey(infoMessageListenerAddress)) {
+ log.warn("Receive a duplicate registration from info message listener on ({})", infoMessageListenerAddress);
+ } else {
+ Future<InfoMessageListenerRpcGateway> infoMessageListenerRpcGatewayFuture = getRpcService().connect(infoMessageListenerAddress, InfoMessageListenerRpcGateway.class);
+
+ infoMessageListenerRpcGatewayFuture.thenAcceptAsync(new AcceptFunction<InfoMessageListenerRpcGateway>() {
+ @Override
+ public void accept(InfoMessageListenerRpcGateway gateway) {
+ log.info("Receive a registration from info message listener on ({})", infoMessageListenerAddress);
+ infoMessageListeners.put(infoMessageListenerAddress, gateway);
+ }
+ }, getMainThreadExecutor());
+
+ infoMessageListenerRpcGatewayFuture.exceptionallyAsync(new ApplyFunction<Throwable, Void>() {
+ @Override
+ public Void apply(Throwable failure) {
+ log.warn("Receive a registration from unreachable info message listener on ({})", infoMessageListenerAddress);
+ return null;
+ }
+ }, getMainThreadExecutor());
+ }
+ }
+
+ /**
+ * Unregisters an infoMessage listener
+ *
+ * @param infoMessageListenerAddress address of infoMessage listener to unregister from this resource manager
+ *
+ */
+ @RpcMethod
+ public void unRegisterInfoMessageListener(final String infoMessageListenerAddress) {
+ infoMessageListeners.remove(infoMessageListenerAddress);
+ }
+
+ /**
+ * Shutdowns cluster
+ *
+ * @param finalStatus
+ * @param optionalDiagnostics
+ */
+ @RpcMethod
+ public void shutDownCluster(final ApplicationStatus finalStatus, final String optionalDiagnostics) {
+ log.info("shut down cluster because application is in {}, diagnostics {}", finalStatus, optionalDiagnostics);
+ shutDownApplication(finalStatus, optionalDiagnostics);
+ }
+
+ /**
+ * This method should be called by the framework once it detects that a currently registered task executor has failed.
+ *
+ * @param resourceID Id of the worker that has failed.
+ * @param message An informational message that explains why the worker failed.
+ */
+ public void notifyWorkerFailed(final ResourceID resourceID, String message) {
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ WorkerType worker = taskExecutorGateways.remove(resourceID);
+ if (worker != null) {
+ // TODO :: suggest failed task executor to stop itself
+ slotManager.notifyTaskManagerFailure(resourceID);
+ }
+ }
+ });
+ }
+
+ /**
+ * Gets the number of currently started TaskManagers.
+ *
+ * @return The number of currently started TaskManagers.
+ */
+ public int getNumberOfStartedTaskManagers() {
+ return taskExecutorGateways.size();
+ }
+
+ /**
+ * Notifies the resource manager of a fatal error.
+ *
+ * <p><b>IMPORTANT:</b> This should not cleanly shut down this master, but exit it in
+ * such a way that a high-availability setting would restart this or fail over
+ * to another master.
+ */
+ public void onFatalError(final String message, final Throwable error) {
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ fatalError(message, error);
+ }
+ });
+ }
+
+ // ------------------------------------------------------------------------
+ // Framework specific behavior
+ // ------------------------------------------------------------------------
+
+ /**
+ * Initializes the framework specific components.
+ *
+ * @throws Exception Exceptions during initialization cause the resource manager to fail.
+ */
+ protected abstract void initialize() throws Exception;
+
+ /**
+ * Callback when a task executor register.
+ *
+ * @param resourceID The worker resource id
+ * @param taskExecutorGateway the task executor gateway
+ */
+ protected abstract WorkerType workerStarted(ResourceID resourceID, TaskExecutorGateway taskExecutorGateway);
+
+ /**
+ * Callback when a resource manager faced a fatal error
+ * @param message
+ * @param error
+ */
+ protected abstract void fatalError(String message, Throwable error);
+
+ /**
+ * The framework specific code for shutting down the application. This should report the
+ * application's final status and shut down the resource manager cleanly.
+ *
+ * This method also needs to make sure all pending containers that are not registered
+ * yet are returned.
+ *
+ * @param finalStatus The application status to report.
+ * @param optionalDiagnostics An optional diagnostics message.
+ */
+ protected abstract void shutDownApplication(ApplicationStatus finalStatus, String optionalDiagnostics);
+
+ // ------------------------------------------------------------------------
+ // Info messaging
+ // ------------------------------------------------------------------------
+
+ public void sendInfoMessage(final String message) {
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ InfoMessage infoMessage = new InfoMessage(message);
+ for (InfoMessageListenerRpcGateway listenerRpcGateway : infoMessageListeners.values()) {
+ listenerRpcGateway
+ .notifyInfoMessage(infoMessage);
+ }
+ }
+ });
+ }
+
private static class JobMasterLeaderListener implements LeaderRetrievalListener {
private final JobID jobID;
@@ -343,5 +508,6 @@ public class ResourceManager extends RpcEndpoint<ResourceManagerGateway> impleme
// TODO
}
}
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/359ee0ad/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
index 484cea7..7c44006 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
@@ -20,6 +20,7 @@ package org.apache.flink.runtime.resourcemanager;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.clusterframework.ApplicationStatus;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.rpc.RpcGateway;
@@ -75,4 +76,26 @@ public interface ResourceManagerGateway extends RpcGateway {
String taskExecutorAddress,
ResourceID resourceID,
@RpcTimeout Time timeout);
+
+ /**
+ * Registers an infoMessage listener
+ *
+ * @param infoMessageListenerAddress address of infoMessage listener to register to this resource manager
+ */
+ void registerInfoMessageListener(String infoMessageListenerAddress);
+
+ /**
+ * Unregisters an infoMessage listener
+ *
+ * @param infoMessageListenerAddress address of infoMessage listener to unregister from this resource manager
+ *
+ */
+ void unRegisterInfoMessageListener(String infoMessageListenerAddress);
+
+ /**
+ * shutdown cluster
+ * @param finalStatus
+ * @param optionalDiagnostics
+ */
+ void shutDownCluster(final ApplicationStatus finalStatus, final String optionalDiagnostics);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/359ee0ad/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
new file mode 100644
index 0000000..84db1ee
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager;
+
+import org.apache.flink.runtime.clusterframework.ApplicationStatus;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.instance.InstanceID;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
+import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
+
+/**
+ * A standalone implementation of the resource manager. Used when the system is started in
+ * standalone mode (via scripts), rather than via a resource framework like YARN or Mesos.
+ */
+public class StandaloneResourceManager extends ResourceManager<ResourceManagerGateway, TaskExecutorRegistration> {
+
+ public StandaloneResourceManager(RpcService rpcService,
+ HighAvailabilityServices highAvailabilityServices,
+ SlotManager slotManager) {
+ super(rpcService, highAvailabilityServices, slotManager);
+ }
+
+ @Override
+ protected void initialize() throws Exception {
+ // nothing to initialize
+ }
+
+ @Override
+ protected void fatalError(final String message, final Throwable error) {
+ log.error("FATAL ERROR IN RESOURCE MANAGER: " + message, error);
+ // kill this process
+ System.exit(EXIT_CODE_FATAL_ERROR);
+ }
+
+ @Override
+ protected TaskExecutorRegistration workerStarted(ResourceID resourceID, TaskExecutorGateway taskExecutorGateway) {
+ InstanceID instanceID = new InstanceID();
+ TaskExecutorRegistration taskExecutorRegistration = new TaskExecutorRegistration(taskExecutorGateway, instanceID);
+ return taskExecutorRegistration;
+ }
+
+ @Override
+ protected void shutDownApplication(ApplicationStatus finalStatus, String optionalDiagnostics) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/359ee0ad/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
index 64a1191..fdb83f5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
@@ -55,7 +55,7 @@ public class ResourceManagerHATest {
highAvailabilityServices.setResourceManagerLeaderElectionService(leaderElectionService);
SlotManager slotManager = mock(SlotManager.class);
- final ResourceManager resourceManager = new ResourceManager(rpcService, highAvailabilityServices, slotManager);
+ final ResourceManager resourceManager = new StandaloneResourceManager(rpcService, highAvailabilityServices, slotManager);
resourceManager.start();
// before grant leadership, resourceManager's leaderId is null
Assert.assertNull(resourceManager.getLeaderSessionID());
http://git-wip-us.apache.org/repos/asf/flink/blob/359ee0ad/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
index 332c093..8f09152 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
@@ -160,7 +160,7 @@ public class ResourceManagerJobMasterTest {
TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
highAvailabilityServices.setResourceManagerLeaderElectionService(resourceManagerLeaderElectionService);
highAvailabilityServices.setJobMasterLeaderRetriever(jobID, jobMasterLeaderRetrievalService);
- ResourceManager resourceManager = new ResourceManager(rpcService, highAvailabilityServices, new SimpleSlotManager());
+ ResourceManager resourceManager = new StandaloneResourceManager(rpcService, highAvailabilityServices, new SimpleSlotManager());
resourceManager.start();
return resourceManager;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/359ee0ad/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
index ed7c7d7..e6d1ed5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
@@ -121,7 +121,7 @@ public class ResourceManagerTaskExecutorTest {
private ResourceManager createAndStartResourceManager(TestingLeaderElectionService rmLeaderElectionService) {
TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
- ResourceManager resourceManager = new ResourceManager(rpcService, highAvailabilityServices, new SimpleSlotManager());
+ ResourceManager resourceManager = new StandaloneResourceManager(rpcService, highAvailabilityServices, new SimpleSlotManager());
resourceManager.start();
return resourceManager;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/359ee0ad/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
index 0232fab..ff25897 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
@@ -35,6 +35,7 @@ import org.apache.flink.runtime.registration.RegistrationResponse;
import org.apache.flink.runtime.resourcemanager.ResourceManager;
import org.apache.flink.runtime.resourcemanager.SlotRequest;
import org.apache.flink.runtime.resourcemanager.SlotRequestReply;
+import org.apache.flink.runtime.resourcemanager.StandaloneResourceManager;
import org.apache.flink.runtime.rpc.TestingSerialRpcService;
import org.apache.flink.runtime.taskexecutor.SlotReport;
import org.apache.flink.runtime.taskexecutor.SlotStatus;
@@ -100,7 +101,7 @@ public class SlotProtocolTest extends TestLogger {
TestingSlotManager slotManager = Mockito.spy(new TestingSlotManager());
ResourceManager resourceManager =
- new ResourceManager(testRpcService, testingHaServices, slotManager);
+ new StandaloneResourceManager(testRpcService, testingHaServices, slotManager);
resourceManager.start();
rmLeaderElectionService.isLeader(rmLeaderID);
@@ -179,7 +180,7 @@ public class SlotProtocolTest extends TestLogger {
TestingSlotManager slotManager = Mockito.spy(new TestingSlotManager());
ResourceManager resourceManager =
- new ResourceManager(testRpcService, testingHaServices, slotManager);
+ new StandaloneResourceManager(testRpcService, testingHaServices, slotManager);
resourceManager.start();
rmLeaderElectionService.isLeader(rmLeaderID);
[27/50] [abbrv] flink git commit: [FLINK-4375] [distributed
coordination] Implement new JobManager creation, initialization,
and basic RPC methods
Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index a2716e5..9f9234f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -18,13 +18,13 @@
package org.apache.flink.runtime.taskexecutor;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.runtime.accumulators.AccumulatorSnapshot;
import org.apache.flink.runtime.blob.BlobCache;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.clusterframework.types.SlotID;
-import org.apache.flink.runtime.instance.InstanceID;
import org.apache.flink.runtime.concurrent.ApplyFunction;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
@@ -33,20 +33,29 @@ import org.apache.flink.runtime.execution.librarycache.LibraryCacheManager;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.executiongraph.PartitionInfo;
import org.apache.flink.runtime.filecache.FileCache;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.instance.InstanceID;
+import org.apache.flink.runtime.io.disk.iomanager.IOManager;
+import org.apache.flink.runtime.io.network.NetworkEnvironment;
import org.apache.flink.runtime.io.network.netty.PartitionStateChecker;
import org.apache.flink.runtime.io.network.partition.ResultPartitionConsumableNotifier;
import org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
import org.apache.flink.runtime.jobmaster.JobMasterGateway;
+import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
+import org.apache.flink.runtime.memory.MemoryManager;
import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.runtime.metrics.MetricRegistry;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRegistered;
-import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRejected;
import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRegistered;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRejected;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
import org.apache.flink.runtime.rpc.FatalErrorHandler;
+import org.apache.flink.runtime.rpc.RpcEndpoint;
+import org.apache.flink.runtime.rpc.RpcMethod;
+import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.taskexecutor.exceptions.CheckpointException;
import org.apache.flink.runtime.taskexecutor.exceptions.PartitionException;
import org.apache.flink.runtime.taskexecutor.exceptions.TaskException;
@@ -60,26 +69,16 @@ import org.apache.flink.runtime.taskmanager.Task;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import org.apache.flink.runtime.taskmanager.TaskManagerActions;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.io.disk.iomanager.IOManager;
-import org.apache.flink.runtime.io.network.NetworkEnvironment;
-import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
-import org.apache.flink.runtime.memory.MemoryManager;
-import org.apache.flink.runtime.rpc.RpcEndpoint;
-import org.apache.flink.runtime.rpc.RpcMethod;
-import org.apache.flink.runtime.rpc.RpcService;
-
import org.apache.flink.util.Preconditions;
-import java.util.HashSet;
-import java.util.Set;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import static org.apache.flink.util.Preconditions.checkArgument;
@@ -276,11 +275,12 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
TaskMetricGroup taskMetricGroup = taskManagerMetricGroup.addTaskForJob(tdd);
InputSplitProvider inputSplitProvider = new RpcInputSplitProvider(
- jobManagerConnection.getJobManagerGateway(),
- tdd.getJobID(),
- tdd.getVertexID(),
- tdd.getExecutionId(),
- taskManagerConfiguration.getTimeout());
+ jobManagerConnection.getJobMasterLeaderId(),
+ jobManagerConnection.getJobManagerGateway(),
+ tdd.getJobID(),
+ tdd.getVertexID(),
+ tdd.getExecutionId(),
+ taskManagerConfiguration.getTimeout());
TaskManagerActions taskManagerActions = jobManagerConnection.getTaskManagerActions();
CheckpointResponder checkpointResponder = jobManagerConnection.getCheckpointResponder();
@@ -580,10 +580,15 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
clearTasks();
}
- private void updateTaskExecutionState(final JobMasterGateway jobMasterGateway, final TaskExecutionState taskExecutionState) {
+ private void updateTaskExecutionState(
+ final UUID jobMasterLeaderId,
+ final JobMasterGateway jobMasterGateway,
+ final TaskExecutionState taskExecutionState)
+ {
final ExecutionAttemptID executionAttemptID = taskExecutionState.getID();
- Future<Acknowledge> futureAcknowledge = jobMasterGateway.updateTaskExecutionState(taskExecutionState);
+ Future<Acknowledge> futureAcknowledge = jobMasterGateway.updateTaskExecutionState(
+ jobMasterLeaderId, taskExecutionState);
futureAcknowledge.exceptionallyAsync(new ApplyFunction<Throwable, Void>() {
@Override
@@ -595,7 +600,11 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}, getMainThreadExecutor());
}
- private void unregisterTaskAndNotifyFinalState(final JobMasterGateway jobMasterGateway, ExecutionAttemptID executionAttemptID) {
+ private void unregisterTaskAndNotifyFinalState(
+ final UUID jobMasterLeaderId,
+ final JobMasterGateway jobMasterGateway,
+ final ExecutionAttemptID executionAttemptID)
+ {
Task task = removeTask(executionAttemptID);
if (task != null) {
@@ -613,13 +622,14 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
AccumulatorSnapshot accumulatorSnapshot = task.getAccumulatorRegistry().getSnapshot();
updateTaskExecutionState(
- jobMasterGateway,
- new TaskExecutionState(
- task.getJobID(),
- task.getExecutionId(),
- task.getExecutionState(),
- task.getFailureCause(),
- accumulatorSnapshot));
+ jobMasterLeaderId,
+ jobMasterGateway,
+ new TaskExecutionState(
+ task.getJobID(),
+ task.getExecutionId(),
+ task.getExecutionState(),
+ task.getFailureCause(),
+ accumulatorSnapshot));
} else {
log.error("Cannot find task with ID {} to unregister.", executionAttemptID);
}
@@ -661,11 +671,14 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
}
- private JobManagerConnection associateWithJobManager(JobMasterGateway jobMasterGateway, int blobPort) {
+ private JobManagerConnection associateWithJobManager(UUID jobMasterLeaderId,
+ JobMasterGateway jobMasterGateway, int blobPort)
+ {
+ Preconditions.checkNotNull(jobMasterLeaderId);
Preconditions.checkNotNull(jobMasterGateway);
Preconditions.checkArgument(blobPort > 0 || blobPort <= 65535, "Blob port is out of range.");
- TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobMasterGateway);
+ TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobMasterLeaderId, jobMasterGateway);
CheckpointResponder checkpointResponder = new RpcCheckpointResponder(jobMasterGateway);
@@ -678,19 +691,21 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
taskManagerConfiguration.getCleanupInterval());
ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = new RpcResultPartitionConsumableNotifier(
- jobMasterGateway,
- getRpcService().getExecutor(),
- taskManagerConfiguration.getTimeout());
+ jobMasterLeaderId,
+ jobMasterGateway,
+ getRpcService().getExecutor(),
+ taskManagerConfiguration.getTimeout());
- PartitionStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobMasterGateway);
+ PartitionStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobMasterLeaderId, jobMasterGateway);
return new JobManagerConnection(
- jobMasterGateway,
- taskManagerActions,
- checkpointResponder,
- libraryCacheManager,
- resultPartitionConsumableNotifier,
- partitionStateChecker);
+ jobMasterLeaderId,
+ jobMasterGateway,
+ taskManagerActions,
+ checkpointResponder,
+ libraryCacheManager,
+ resultPartitionConsumableNotifier,
+ partitionStateChecker);
}
private void disassociateFromJobManager(JobManagerConnection jobManagerConnection) throws IOException {
@@ -782,9 +797,11 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
}
private class TaskManagerActionsImpl implements TaskManagerActions {
+ private final UUID jobMasterLeaderId;
private final JobMasterGateway jobMasterGateway;
- private TaskManagerActionsImpl(JobMasterGateway jobMasterGateway) {
+ private TaskManagerActionsImpl(UUID jobMasterLeaderId, JobMasterGateway jobMasterGateway) {
+ this.jobMasterLeaderId = Preconditions.checkNotNull(jobMasterLeaderId);
this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
}
@@ -793,7 +810,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
runAsync(new Runnable() {
@Override
public void run() {
- unregisterTaskAndNotifyFinalState(jobMasterGateway, executionAttemptID);
+ unregisterTaskAndNotifyFinalState(jobMasterLeaderId, jobMasterGateway, executionAttemptID);
}
});
}
@@ -816,7 +833,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
@Override
public void updateTaskExecutionState(final TaskExecutionState taskExecutionState) {
- TaskExecutor.this.updateTaskExecutionState(jobMasterGateway, taskExecutionState);
+ TaskExecutor.this.updateTaskExecutionState(jobMasterLeaderId, jobMasterGateway, taskExecutionState);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java
index 246c11d..9669da0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcCheckpointResponder.java
@@ -51,6 +51,6 @@ public class RpcCheckpointResponder implements CheckpointResponder {
@Override
public void declineCheckpoint(JobID jobID, ExecutionAttemptID executionAttemptID, CheckpointMetaData checkpoint) {
- checkpointCoordinatorGateway.declineCheckpoint(jobID, executionAttemptID, checkpoint);
+ checkpointCoordinatorGateway.declineCheckpoint(jobID, executionAttemptID, checkpoint.getCheckpointId());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcInputSplitProvider.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcInputSplitProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcInputSplitProvider.java
index 4850d63..3b9da48 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcInputSplitProvider.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcInputSplitProvider.java
@@ -31,7 +31,10 @@ import org.apache.flink.runtime.jobmaster.SerializedInputSplit;
import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.Preconditions;
+import java.util.UUID;
+
public class RpcInputSplitProvider implements InputSplitProvider {
+ private final UUID jobMasterLeaderId;
private final JobMasterGateway jobMasterGateway;
private final JobID jobID;
private final JobVertexID jobVertexID;
@@ -39,11 +42,13 @@ public class RpcInputSplitProvider implements InputSplitProvider {
private final Time timeout;
public RpcInputSplitProvider(
+ UUID jobMasterLeaderId,
JobMasterGateway jobMasterGateway,
JobID jobID,
JobVertexID jobVertexID,
ExecutionAttemptID executionAttemptID,
Time timeout) {
+ this.jobMasterLeaderId = Preconditions.checkNotNull(jobMasterLeaderId);
this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
this.jobID = Preconditions.checkNotNull(jobID);
this.jobVertexID = Preconditions.checkNotNull(jobVertexID);
@@ -56,7 +61,8 @@ public class RpcInputSplitProvider implements InputSplitProvider {
public InputSplit getNextInputSplit(ClassLoader userCodeClassLoader) throws InputSplitProviderException {
Preconditions.checkNotNull(userCodeClassLoader);
- Future<SerializedInputSplit> futureInputSplit = jobMasterGateway.requestNextInputSplit(jobVertexID, executionAttemptID);
+ Future<SerializedInputSplit> futureInputSplit = jobMasterGateway.requestNextInputSplit(
+ jobMasterLeaderId, jobVertexID, executionAttemptID);
try {
SerializedInputSplit serializedInputSplit = futureInputSplit.get(timeout.getSize(), timeout.getUnit());
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcPartitionStateChecker.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcPartitionStateChecker.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcPartitionStateChecker.java
index ab111ad..1c91b87 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcPartitionStateChecker.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcPartitionStateChecker.java
@@ -28,11 +28,15 @@ import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobmaster.JobMasterGateway;
import org.apache.flink.util.Preconditions;
+import java.util.UUID;
+
public class RpcPartitionStateChecker implements PartitionStateChecker {
+ private final UUID jobMasterLeaderId;
private final JobMasterGateway jobMasterGateway;
- public RpcPartitionStateChecker(JobMasterGateway jobMasterGateway) {
+ public RpcPartitionStateChecker(UUID jobMasterLeaderId, JobMasterGateway jobMasterGateway) {
+ this.jobMasterLeaderId = Preconditions.checkNotNull(jobMasterLeaderId);
this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
}
@@ -43,6 +47,6 @@ public class RpcPartitionStateChecker implements PartitionStateChecker {
IntermediateDataSetID resultId,
ResultPartitionID partitionId) {
- return jobMasterGateway.requestPartitionState(partitionId, executionId, resultId);
+ return jobMasterGateway.requestPartitionState(jobMasterLeaderId, partitionId, executionId, resultId);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcResultPartitionConsumableNotifier.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcResultPartitionConsumableNotifier.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcResultPartitionConsumableNotifier.java
index 29ad3b6..cf01d5a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcResultPartitionConsumableNotifier.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/rpc/RpcResultPartitionConsumableNotifier.java
@@ -31,27 +31,32 @@ import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.UUID;
import java.util.concurrent.Executor;
public class RpcResultPartitionConsumableNotifier implements ResultPartitionConsumableNotifier {
private static final Logger LOG = LoggerFactory.getLogger(RpcResultPartitionConsumableNotifier.class);
+ private final UUID jobMasterLeaderId;
private final JobMasterGateway jobMasterGateway;
private final Executor executor;
private final Time timeout;
public RpcResultPartitionConsumableNotifier(
+ UUID jobMasterLeaderId,
JobMasterGateway jobMasterGateway,
Executor executor,
Time timeout) {
+ this.jobMasterLeaderId = Preconditions.checkNotNull(jobMasterLeaderId);
this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
this.executor = Preconditions.checkNotNull(executor);
this.timeout = Preconditions.checkNotNull(timeout);
}
@Override
public void notifyPartitionConsumable(JobID jobId, ResultPartitionID partitionId, final TaskActions taskActions) {
- Future<Acknowledge> acknowledgeFuture = jobMasterGateway.scheduleOrUpdateConsumers(partitionId, timeout);
+ Future<Acknowledge> acknowledgeFuture = jobMasterGateway.scheduleOrUpdateConsumers(
+ jobMasterLeaderId, partitionId, timeout);
acknowledgeFuture.exceptionallyAsync(new ApplyFunction<Throwable, Void>() {
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
index a0ca3a1..8394902 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
@@ -659,6 +659,12 @@ object AkkaUtils {
}
}
+ def formatDurationParingErrorMessage: String = {
+ "Duration format must be \"val unit\", where 'val' is a number and 'unit' is " +
+ "(d|day)|(h|hour)|(min|minute)|s|sec|second)|(ms|milli|millisecond)|"+
+ "(�s|micro|microsecond)|(ns|nano|nanosecond)"
+ }
+
/** Returns the protocol field for the URL of the remote actor system given the user configuration
*
* @param config instance containing the user provided configuration values
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
index faf69cc..a255027 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
@@ -19,11 +19,15 @@
package org.apache.flink.runtime.highavailability;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.blob.BlobStore;
+import org.apache.flink.runtime.blob.VoidBlobStore;
import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
+import org.apache.flink.runtime.highavailability.nonha.NonHaRegistry;
import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -140,4 +144,14 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
}
}
+
+ @Override
+ public RunningJobsRegistry getRunningJobsRegistry() throws Exception {
+ return new NonHaRegistry();
+ }
+
+ @Override
+ public BlobStore createBlobStore() throws IOException {
+ return new VoidBlobStore();
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
index 30dfef5..f709cbd 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerMockTest.java
@@ -21,14 +21,21 @@ package org.apache.flink.runtime.jobmaster;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.blob.BlobStore;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.RunningJobsRegistry;
import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobVertex;
import org.apache.flink.runtime.jobmanager.OnCompletionActions;
import org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcService;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
@@ -61,11 +68,19 @@ public class JobManagerRunnerMockTest {
private TestingOnCompletionActions jobCompletion;
+ private BlobStore blobStore;
+
+ private RunningJobsRegistry runningJobsRegistry;
+
@Before
public void setUp() throws Exception {
+ RpcService mockRpc = mock(RpcService.class);
+ when(mockRpc.getAddress()).thenReturn("localhost");
+
jobManager = mock(JobMaster.class);
jobManagerGateway = mock(JobMasterGateway.class);
when(jobManager.getSelf()).thenReturn(jobManagerGateway);
+ when(jobManager.getRpcService()).thenReturn(mockRpc);
PowerMockito.whenNew(JobMaster.class).withAnyArguments().thenReturn(jobManager);
@@ -74,19 +89,25 @@ public class JobManagerRunnerMockTest {
leaderElectionService = mock(LeaderElectionService.class);
when(leaderElectionService.hasLeadership()).thenReturn(true);
- submittedJobGraphStore = mock(SubmittedJobGraphStore.class);
- when(submittedJobGraphStore.contains(any(JobID.class))).thenReturn(true);
+ runningJobsRegistry = mock(RunningJobsRegistry.class);
+ when(runningJobsRegistry.isJobRunning(any(JobID.class))).thenReturn(true);
+ blobStore = mock(BlobStore.class);
+
HighAvailabilityServices haServices = mock(HighAvailabilityServices.class);
when(haServices.getJobManagerLeaderElectionService(any(JobID.class))).thenReturn(leaderElectionService);
when(haServices.getSubmittedJobGraphStore()).thenReturn(submittedJobGraphStore);
+ when(haServices.createBlobStore()).thenReturn(blobStore);
+ when(haServices.getRunningJobsRegistry()).thenReturn(runningJobsRegistry);
runner = PowerMockito.spy(new JobManagerRunner(
- new JobGraph("test"),
+ new JobGraph("test", new JobVertex("vertex")),
mock(Configuration.class),
- mock(RpcService.class),
+ mockRpc,
haServices,
- mock(JobManagerServices.class),
+ JobManagerServices.fromConfiguration(new Configuration(), haServices),
+ new MetricRegistry(MetricRegistryConfiguration.defaultMetricRegistryConfiguration()),
+ jobCompletion,
jobCompletion));
}
@@ -94,25 +115,26 @@ public class JobManagerRunnerMockTest {
public void tearDown() throws Exception {
}
+ @Ignore
@Test
public void testStartAndShutdown() throws Exception {
runner.start();
- verify(jobManager).init();
- verify(jobManager).start();
verify(leaderElectionService).start(runner);
assertTrue(!jobCompletion.isJobFinished());
assertTrue(!jobCompletion.isJobFailed());
+ verify(jobManager).start(any(UUID.class));
+
runner.shutdown();
verify(leaderElectionService).stop();
verify(jobManager).shutDown();
}
+ @Ignore
@Test
public void testShutdownBeforeGrantLeadership() throws Exception {
runner.start();
- verify(jobManager).init();
verify(jobManager).start();
verify(leaderElectionService).start(runner);
@@ -129,13 +151,14 @@ public class JobManagerRunnerMockTest {
}
+ @Ignore
@Test
public void testJobFinished() throws Exception {
runner.start();
UUID leaderSessionID = UUID.randomUUID();
runner.grantLeadership(leaderSessionID);
- verify(jobManagerGateway).startJob(leaderSessionID);
+ verify(jobManager).start(leaderSessionID);
assertTrue(!jobCompletion.isJobFinished());
// runner been told by JobManager that job is finished
@@ -148,13 +171,14 @@ public class JobManagerRunnerMockTest {
assertTrue(runner.isShutdown());
}
+ @Ignore
@Test
public void testJobFailed() throws Exception {
runner.start();
UUID leaderSessionID = UUID.randomUUID();
runner.grantLeadership(leaderSessionID);
- verify(jobManagerGateway).startJob(leaderSessionID);
+ verify(jobManager).start(leaderSessionID);
assertTrue(!jobCompletion.isJobFinished());
// runner been told by JobManager that job is failed
@@ -166,39 +190,41 @@ public class JobManagerRunnerMockTest {
assertTrue(runner.isShutdown());
}
+ @Ignore
@Test
public void testLeadershipRevoked() throws Exception {
runner.start();
UUID leaderSessionID = UUID.randomUUID();
runner.grantLeadership(leaderSessionID);
- verify(jobManagerGateway).startJob(leaderSessionID);
+ verify(jobManager).start(leaderSessionID);
assertTrue(!jobCompletion.isJobFinished());
runner.revokeLeadership();
- verify(jobManagerGateway).suspendJob(any(Throwable.class));
+ verify(jobManager).suspendExecution(any(Throwable.class));
assertFalse(runner.isShutdown());
}
+ @Ignore
@Test
public void testRegainLeadership() throws Exception {
runner.start();
UUID leaderSessionID = UUID.randomUUID();
runner.grantLeadership(leaderSessionID);
- verify(jobManagerGateway).startJob(leaderSessionID);
+ verify(jobManager).start(leaderSessionID);
assertTrue(!jobCompletion.isJobFinished());
runner.revokeLeadership();
- verify(jobManagerGateway).suspendJob(any(Throwable.class));
+ verify(jobManager).suspendExecution(any(Throwable.class));
assertFalse(runner.isShutdown());
UUID leaderSessionID2 = UUID.randomUUID();
runner.grantLeadership(leaderSessionID2);
- verify(jobManagerGateway).startJob(leaderSessionID2);
+ verify(jobManager).start(leaderSessionID2);
}
- private static class TestingOnCompletionActions implements OnCompletionActions {
+ private static class TestingOnCompletionActions implements OnCompletionActions, FatalErrorHandler {
private volatile JobExecutionResult result;
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java
new file mode 100644
index 0000000..174422f
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+public class JobManagerRunnerTest {
+
+ // TODO: Test that
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5f62d3b7/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSinkTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSinkTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSinkTaskTest.java
index a41c25b..685440b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSinkTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSinkTaskTest.java
@@ -58,7 +58,7 @@ import static org.junit.Assert.assertFalse;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Task.class, ResultPartitionWriter.class})
-@PowerMockIgnore({"javax.management.*", "com.sun.jndi.*"})
+@PowerMockIgnore({"javax.management.*", "com.sun.jndi.*", "org.apache.log4j.*"})
public class DataSinkTaskTest extends TaskTestBase {
private static final Logger LOG = LoggerFactory.getLogger(DataSinkTaskTest.class);
[42/50] [abbrv] flink git commit: [FLINK-4882] [flip-6] Remove
exceptions from HighAvailabilityServices where not necessary
Posted by tr...@apache.org.
[FLINK-4882] [flip-6] Remove exceptions from HighAvailabilityServices where not necessary
Cleanup of the interface HighAvailabilityServices so that only methods which really throw an
exception have an exception clause defined.
This closes #2679.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d2d0b127
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d2d0b127
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d2d0b127
Branch: refs/heads/flip-6
Commit: d2d0b12752dc00885f34ed08362cb9fa9d7105a7
Parents: 5c55266
Author: Till Rohrmann <tr...@apache.org>
Authored: Wed Oct 19 14:09:31 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:34 2016 +0100
----------------------------------------------------------------------
.../highavailability/EmbeddedNonHaServices.java | 4 +--
.../HighAvailabilityServices.java | 33 +++++++++++++++-----
.../runtime/highavailability/NonHaServices.java | 4 +--
.../highavailability/ZookeeperHaServices.java | 12 +++----
.../nonha/AbstractNonHaServices.java | 10 +++---
.../flink/runtime/jobmaster/JobMaster.java | 17 ++--------
.../TestingHighAvailabilityServices.java | 14 ++++-----
7 files changed, 49 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/d2d0b127/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java
index 58da287..523218e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/EmbeddedNonHaServices.java
@@ -43,12 +43,12 @@ public class EmbeddedNonHaServices extends AbstractNonHaServices implements High
// ------------------------------------------------------------------------
@Override
- public LeaderRetrievalService getResourceManagerLeaderRetriever() throws Exception {
+ public LeaderRetrievalService getResourceManagerLeaderRetriever() {
return resourceManagerLeaderService.createLeaderRetrievalService();
}
@Override
- public LeaderElectionService getResourceManagerLeaderElectionService() throws Exception {
+ public LeaderElectionService getResourceManagerLeaderElectionService() {
return resourceManagerLeaderService.createLeaderElectionService();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d2d0b127/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
index f6db682..360de7b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServices.java
@@ -43,50 +43,67 @@ public interface HighAvailabilityServices {
/**
* Gets the leader retriever for the cluster's resource manager.
*/
- LeaderRetrievalService getResourceManagerLeaderRetriever() throws Exception;
+ LeaderRetrievalService getResourceManagerLeaderRetriever();
/**
* Gets the leader retriever for the job JobMaster which is responsible for the given job
*
* @param jobID The identifier of the job.
- * @return
- * @throws Exception
+ * @return Leader retrieval service to retrieve the job manager for the given job
*/
- LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception;
+ LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID);
/**
* Gets the leader election service for the cluster's resource manager.
+ *
+ * @return Leader election service for the resource manager leader election
*/
- LeaderElectionService getResourceManagerLeaderElectionService() throws Exception;
+ LeaderElectionService getResourceManagerLeaderElectionService();
/**
* Gets the leader election service for the given job.
*
* @param jobID The identifier of the job running the election.
+ * @return Leader election service for the job manager leader election
*/
- LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) throws Exception;
+ LeaderElectionService getJobManagerLeaderElectionService(JobID jobID);
/**
* Gets the checkpoint recovery factory for the job manager
+ *
+ * @return Checkpoint recovery factory
*/
- CheckpointRecoveryFactory getCheckpointRecoveryFactory() throws Exception;
+ CheckpointRecoveryFactory getCheckpointRecoveryFactory();
/**
* Gets the submitted job graph store for the job manager
+ *
+ * @return Submitted job graph store
+ * @throws Exception if the submitted job graph store could not be created
*/
SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception;
/**
* Gets the registry that holds information about whether jobs are currently running.
+ *
+ * @return Running job registry to retrieve running jobs
*/
- RunningJobsRegistry getRunningJobsRegistry() throws Exception;
+ RunningJobsRegistry getRunningJobsRegistry();
/**
* Creates the BLOB store in which BLOBs are stored in a highly-available fashion.
+ *
+ * @return Blob store
+ * @throws IOException if the blob store could not be created
*/
BlobStore createBlobStore() throws IOException;
// ------------------------------------------------------------------------
+ /**
+ * Shut the high availability service down.
+ *
+ * @throws Exception if the shut down fails
+ */
void shutdown() throws Exception;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d2d0b127/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
index 107cbd0..75f44ed 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/NonHaServices.java
@@ -57,12 +57,12 @@ public class NonHaServices extends AbstractNonHaServices implements HighAvailabi
// ------------------------------------------------------------------------
@Override
- public LeaderRetrievalService getResourceManagerLeaderRetriever() throws Exception {
+ public LeaderRetrievalService getResourceManagerLeaderRetriever() {
return new StandaloneLeaderRetrievalService(resourceManagerAddress, new UUID(0, 0));
}
@Override
- public LeaderElectionService getResourceManagerLeaderElectionService() throws Exception {
+ public LeaderElectionService getResourceManagerLeaderElectionService() {
return new StandaloneLeaderElectionService();
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d2d0b127/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
index a9d2610..07c5011 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
@@ -107,27 +107,27 @@ public class ZookeeperHaServices implements HighAvailabilityServices {
// ------------------------------------------------------------------------
@Override
- public LeaderRetrievalService getResourceManagerLeaderRetriever() throws Exception {
+ public LeaderRetrievalService getResourceManagerLeaderRetriever() {
return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, RESOURCE_MANAGER_LEADER_PATH);
}
@Override
- public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception {
+ public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, getPathForJobManager(jobID));
}
@Override
- public LeaderElectionService getResourceManagerLeaderElectionService() throws Exception {
+ public LeaderElectionService getResourceManagerLeaderElectionService() {
return ZooKeeperUtils.createLeaderElectionService(client, configuration, RESOURCE_MANAGER_LEADER_PATH);
}
@Override
- public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) throws Exception {
+ public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
return ZooKeeperUtils.createLeaderElectionService(client, configuration, getPathForJobManager(jobID));
}
@Override
- public CheckpointRecoveryFactory getCheckpointRecoveryFactory() throws Exception {
+ public CheckpointRecoveryFactory getCheckpointRecoveryFactory() {
return new ZooKeeperCheckpointRecoveryFactory(client, configuration);
}
@@ -137,7 +137,7 @@ public class ZookeeperHaServices implements HighAvailabilityServices {
}
@Override
- public RunningJobsRegistry getRunningJobsRegistry() throws Exception {
+ public RunningJobsRegistry getRunningJobsRegistry() {
throw new UnsupportedOperationException("not yet implemented");
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d2d0b127/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/AbstractNonHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/AbstractNonHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/AbstractNonHaServices.java
index 8c15a52..237727f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/AbstractNonHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/AbstractNonHaServices.java
@@ -72,7 +72,7 @@ public abstract class AbstractNonHaServices implements HighAvailabilityServices
// ------------------------------------------------------------------------
@Override
- public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception {
+ public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
checkNotNull(jobID);
synchronized (lock) {
@@ -83,7 +83,7 @@ public abstract class AbstractNonHaServices implements HighAvailabilityServices
}
@Override
- public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) throws Exception {
+ public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
checkNotNull(jobID);
synchronized (lock) {
@@ -104,19 +104,19 @@ public abstract class AbstractNonHaServices implements HighAvailabilityServices
}
@Override
- public CheckpointRecoveryFactory getCheckpointRecoveryFactory() throws Exception {
+ public CheckpointRecoveryFactory getCheckpointRecoveryFactory() {
checkNotShutdown();
return new StandaloneCheckpointRecoveryFactory();
}
@Override
- public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
+ public SubmittedJobGraphStore getSubmittedJobGraphStore() {
checkNotShutdown();
return new StandaloneSubmittedJobGraphStore();
}
@Override
- public RunningJobsRegistry getRunningJobsRegistry() throws Exception {
+ public RunningJobsRegistry getRunningJobsRegistry() {
checkNotShutdown();
return runningJobsRegistry;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d2d0b127/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 5a7c9a1..a9ac1fe 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -33,7 +33,6 @@ import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory;
import org.apache.flink.runtime.checkpoint.CompletedCheckpoint;
import org.apache.flink.runtime.checkpoint.SubtaskState;
import org.apache.flink.runtime.client.JobExecutionException;
-import org.apache.flink.runtime.client.JobSubmissionException;
import org.apache.flink.runtime.client.SerializedJobExecutionResult;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
@@ -229,21 +228,9 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
log.info("Using restart strategy {} for {} ({}).", restartStrategy, jobName, jid);
- CheckpointRecoveryFactory checkpointRecoveryFactory;
- try {
- checkpointRecoveryFactory = highAvailabilityServices.getCheckpointRecoveryFactory();
- } catch (Exception e) {
- log.error("Could not create the access to highly-available checkpoint storage.", e);
- throw new Exception("Could not create the access to highly-available checkpoint storage.", e);
- }
+ CheckpointRecoveryFactory checkpointRecoveryFactory = highAvailabilityServices.getCheckpointRecoveryFactory();
- try {
- resourceManagerLeaderRetriever = highAvailabilityServices.getResourceManagerLeaderRetriever();
- } catch (Exception e) {
- log.error("Could not get the resource manager leader retriever.", e);
- throw new JobSubmissionException(jobGraph.getJobID(),
- "Could not get the resource manager leader retriever.", e);
- }
+ resourceManagerLeaderRetriever = highAvailabilityServices.getResourceManagerLeaderRetriever();
this.executionGraph = ExecutionGraphBuilder.buildGraph(
null,
http://git-wip-us.apache.org/repos/asf/flink/blob/d2d0b127/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
index 877812b..e0f71ee 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
@@ -81,7 +81,7 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
// ------------------------------------------------------------------------
@Override
- public LeaderRetrievalService getResourceManagerLeaderRetriever() throws Exception {
+ public LeaderRetrievalService getResourceManagerLeaderRetriever() {
LeaderRetrievalService service = this.resourceManagerLeaderRetriever;
if (service != null) {
return service;
@@ -91,7 +91,7 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
}
@Override
- public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception {
+ public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
LeaderRetrievalService service = this.jobMasterLeaderRetrievers.get(jobID);
if (service != null) {
return service;
@@ -101,7 +101,7 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
}
@Override
- public LeaderElectionService getResourceManagerLeaderElectionService() throws Exception {
+ public LeaderElectionService getResourceManagerLeaderElectionService() {
LeaderElectionService service = resourceManagerLeaderElectionService;
if (service != null) {
@@ -112,7 +112,7 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
}
@Override
- public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) throws Exception {
+ public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
LeaderElectionService service = this.jobManagerLeaderElectionServices.get(jobID);
if (service != null) {
@@ -123,7 +123,7 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
}
@Override
- public CheckpointRecoveryFactory getCheckpointRecoveryFactory() throws Exception {
+ public CheckpointRecoveryFactory getCheckpointRecoveryFactory() {
CheckpointRecoveryFactory factory = checkpointRecoveryFactory;
if (factory != null) {
@@ -134,7 +134,7 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
}
@Override
- public SubmittedJobGraphStore getSubmittedJobGraphStore() throws Exception {
+ public SubmittedJobGraphStore getSubmittedJobGraphStore() {
SubmittedJobGraphStore store = submittedJobGraphStore;
if (store != null) {
@@ -146,7 +146,7 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
}
@Override
- public RunningJobsRegistry getRunningJobsRegistry() throws Exception {
+ public RunningJobsRegistry getRunningJobsRegistry() {
return new NonHaRegistry();
}
[03/50] [abbrv] flink git commit: [FLINK-4530] [rpc] Generalize
TaskExecutorToResourceManagerConnection to be reusable
Posted by tr...@apache.org.
[FLINK-4530] [rpc] Generalize TaskExecutorToResourceManagerConnection to be reusable
This closes #2520
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/20e4e12d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/20e4e12d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/20e4e12d
Branch: refs/heads/flip-6
Commit: 20e4e12d6ff1c9200b44828f141e4878c53df7de
Parents: 5647006
Author: zhuhaifengleon <zh...@gmail.com>
Authored: Mon Sep 26 17:43:44 2016 +0800
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:27 2016 +0100
----------------------------------------------------------------------
.../JobMasterToResourceManagerConnection.java | 117 +++++++++++
.../registration/RegisteredRpcConnection.java | 192 +++++++++++++++++++
.../runtime/taskexecutor/TaskExecutor.java | 4 +-
...TaskExecutorToResourceManagerConnection.java | 127 +++---------
.../RegisteredRpcConnectionTest.java | 183 ++++++++++++++++++
.../registration/RetryingRegistrationTest.java | 6 +-
6 files changed, 519 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/20e4e12d/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterToResourceManagerConnection.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterToResourceManagerConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterToResourceManagerConnection.java
new file mode 100644
index 0000000..71fce8c
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterToResourceManagerConnection.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.registration.RegisteredRpcConnection;
+import org.apache.flink.runtime.registration.RegistrationResponse;
+import org.apache.flink.runtime.registration.RetryingRegistration;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
+import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.runtime.concurrent.Future;
+
+import org.slf4j.Logger;
+
+import java.util.UUID;
+import java.util.concurrent.Executor;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * The connection between a JobMaster and the ResourceManager.
+ */
+public class JobMasterToResourceManagerConnection
+ extends RegisteredRpcConnection<ResourceManagerGateway, JobMasterRegistrationSuccess> {
+
+ /** the JobMaster whose connection to the ResourceManager this represents */
+ private final JobMaster jobMaster;
+
+ private final JobID jobID;
+
+ private final UUID jobMasterLeaderId;
+
+ public JobMasterToResourceManagerConnection(
+ Logger log,
+ JobID jobID,
+ JobMaster jobMaster,
+ UUID jobMasterLeaderId,
+ String resourceManagerAddress,
+ UUID resourceManagerLeaderId,
+ Executor executor) {
+
+ super(log, resourceManagerAddress, resourceManagerLeaderId, executor);
+ this.jobMaster = checkNotNull(jobMaster);
+ this.jobID = checkNotNull(jobID);
+ this.jobMasterLeaderId = checkNotNull(jobMasterLeaderId);
+ }
+
+ @Override
+ protected RetryingRegistration<ResourceManagerGateway, JobMasterRegistrationSuccess> generateRegistration() {
+ return new JobMasterToResourceManagerConnection.ResourceManagerRegistration(
+ log, jobMaster.getRpcService(),
+ getTargetAddress(), getTargetLeaderId(),
+ jobMaster.getAddress(),jobID, jobMasterLeaderId);
+ }
+
+ @Override
+ protected void onRegistrationSuccess(JobMasterRegistrationSuccess success) {
+ }
+
+ @Override
+ protected void onRegistrationFailure(Throwable failure) {
+ }
+
+ // ------------------------------------------------------------------------
+ // Utilities
+ // ------------------------------------------------------------------------
+
+ private static class ResourceManagerRegistration
+ extends RetryingRegistration<ResourceManagerGateway, JobMasterRegistrationSuccess> {
+
+ private final String jobMasterAddress;
+
+ private final JobID jobID;
+
+ private final UUID jobMasterLeaderId;
+
+ ResourceManagerRegistration(
+ Logger log,
+ RpcService rpcService,
+ String targetAddress,
+ UUID leaderId,
+ String jobMasterAddress,
+ JobID jobID,
+ UUID jobMasterLeaderId) {
+
+ super(log, rpcService, "ResourceManager", ResourceManagerGateway.class, targetAddress, leaderId);
+ this.jobMasterAddress = checkNotNull(jobMasterAddress);
+ this.jobID = checkNotNull(jobID);
+ this.jobMasterLeaderId = checkNotNull(jobMasterLeaderId);
+ }
+
+ @Override
+ protected Future<RegistrationResponse> invokeRegistration(
+ ResourceManagerGateway gateway, UUID leaderId, long timeoutMillis) throws Exception {
+
+ Time timeout = Time.milliseconds(timeoutMillis);
+ return gateway.registerJobMaster(leaderId, jobMasterLeaderId,jobMasterAddress, jobID, timeout);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/20e4e12d/flink-runtime/src/main/java/org/apache/flink/runtime/registration/RegisteredRpcConnection.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/registration/RegisteredRpcConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/registration/RegisteredRpcConnection.java
new file mode 100644
index 0000000..76093b0
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/registration/RegisteredRpcConnection.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.registration;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.runtime.rpc.RpcGateway;
+import org.apache.flink.runtime.concurrent.AcceptFunction;
+import org.apache.flink.runtime.concurrent.ApplyFunction;
+import org.apache.flink.runtime.concurrent.Future;
+
+import org.slf4j.Logger;
+
+import java.util.UUID;
+import java.util.concurrent.Executor;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
+/**
+ * This utility class implements the basis of RPC connecting from one component to another component,
+ * for example the RPC connection from TaskExecutor to ResourceManager.
+ * This {@code RegisteredRpcConnection} implements registration and get target gateway .
+ *
+ * <p>The registration gives access to a future that is completed upon successful registration.
+ * The RPC connection can be closed, for example when the target where it tries to register
+ * at looses leader status.
+ *
+ * @param <Gateway> The type of the gateway to connect to.
+ * @param <Success> The type of the successful registration responses.
+ */
+public abstract class RegisteredRpcConnection<Gateway extends RpcGateway, Success extends RegistrationResponse.Success> {
+
+ /** the logger for all log messages of this class */
+ protected final Logger log;
+
+ /** the target component leaderID, for example the ResourceManager leaderID */
+ private final UUID targetLeaderId;
+
+ /** the target component Address, for example the ResourceManager Address */
+ private final String targetAddress;
+
+ /** Execution context to be used to execute the on complete action of the ResourceManagerRegistration */
+ private final Executor executor;
+
+ /** the Registration of this RPC connection */
+ private RetryingRegistration<Gateway, Success> pendingRegistration;
+
+ /** the gateway to register, it's null until the registration is completed */
+ private volatile Gateway targetGateway;
+
+ /** flag indicating that the RPC connection is closed */
+ private volatile boolean closed;
+
+ // ------------------------------------------------------------------------
+
+ public RegisteredRpcConnection(
+ Logger log,
+ String targetAddress,
+ UUID targetLeaderId,
+ Executor executor)
+ {
+ this.log = checkNotNull(log);
+ this.targetAddress = checkNotNull(targetAddress);
+ this.targetLeaderId = checkNotNull(targetLeaderId);
+ this.executor = checkNotNull(executor);
+ }
+
+ // ------------------------------------------------------------------------
+ // Life cycle
+ // ------------------------------------------------------------------------
+
+ @SuppressWarnings("unchecked")
+ public void start() {
+ checkState(!closed, "The RPC connection is already closed");
+ checkState(!isConnected() && pendingRegistration == null, "The RPC connection is already started");
+
+ pendingRegistration = checkNotNull(generateRegistration());
+ pendingRegistration.startRegistration();
+
+ Future<Tuple2<Gateway, Success>> future = pendingRegistration.getFuture();
+
+ future.thenAcceptAsync(new AcceptFunction<Tuple2<Gateway, Success>>() {
+ @Override
+ public void accept(Tuple2<Gateway, Success> result) {
+ targetGateway = result.f0;
+ onRegistrationSuccess(result.f1);
+ }
+ }, executor);
+
+ // this future should only ever fail if there is a bug, not if the registration is declined
+ future.exceptionallyAsync(new ApplyFunction<Throwable, Void>() {
+ @Override
+ public Void apply(Throwable failure) {
+ onRegistrationFailure(failure);
+ return null;
+ }
+ }, executor);
+ }
+
+ /**
+ * This method generate a specific Registration, for example TaskExecutor Registration at the ResourceManager
+ */
+ protected abstract RetryingRegistration<Gateway, Success> generateRegistration();
+
+ /**
+ * This method handle the Registration Response
+ */
+ protected abstract void onRegistrationSuccess(Success success);
+
+ /**
+ * This method handle the Registration failure
+ */
+ protected abstract void onRegistrationFailure(Throwable failure);
+
+ /**
+ * close connection
+ */
+ public void close() {
+ closed = true;
+
+ // make sure we do not keep re-trying forever
+ if (pendingRegistration != null) {
+ pendingRegistration.cancel();
+ }
+ }
+
+ public boolean isClosed() {
+ return closed;
+ }
+
+ // ------------------------------------------------------------------------
+ // Properties
+ // ------------------------------------------------------------------------
+
+ public UUID getTargetLeaderId() {
+ return targetLeaderId;
+ }
+
+ public String getTargetAddress() {
+ return targetAddress;
+ }
+
+ /**
+ * Gets the RegisteredGateway. This returns null until the registration is completed.
+ */
+ public Gateway getTargetGateway() {
+ return targetGateway;
+ }
+
+ public boolean isConnected() {
+ return targetGateway != null;
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public String toString() {
+ String connectionInfo = "(ADDRESS: " + targetAddress + " LEADERID: " + targetLeaderId + ")";
+
+ if (isConnected()) {
+ connectionInfo = "RPC connection to " + targetGateway.getClass().getSimpleName() + " " + connectionInfo;
+ } else {
+ connectionInfo = "RPC connection to " + connectionInfo;
+ }
+
+ if (isClosed()) {
+ connectionInfo = connectionInfo + " is closed";
+ } else if (isConnected()){
+ connectionInfo = connectionInfo + " is established";
+ } else {
+ connectionInfo = connectionInfo + " is connecting";
+ }
+
+ return connectionInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/20e4e12d/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index 9e3c3b9..9d9ad2a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -178,12 +178,12 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
if (newLeaderAddress != null) {
// the resource manager switched to a new leader
log.info("ResourceManager leader changed from {} to {}. Registering at new leader.",
- resourceManagerConnection.getResourceManagerAddress(), newLeaderAddress);
+ resourceManagerConnection.getTargetAddress(), newLeaderAddress);
}
else {
// address null means that the current leader is lost without a new leader being there, yet
log.info("Current ResourceManager {} lost leader status. Waiting for new ResourceManager leader.",
- resourceManagerConnection.getResourceManagerAddress());
+ resourceManagerConnection.getTargetAddress());
}
// drop the current connection or connection attempt
http://git-wip-us.apache.org/repos/asf/flink/blob/20e4e12d/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java
index 647359d..b4b3bae 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorToResourceManagerConnection.java
@@ -19,16 +19,14 @@
package org.apache.flink.runtime.taskexecutor;
import org.apache.flink.api.common.time.Time;
-import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.concurrent.AcceptFunction;
-import org.apache.flink.runtime.concurrent.ApplyFunction;
-import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.instance.InstanceID;
+import org.apache.flink.runtime.registration.RegisteredRpcConnection;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.registration.RegistrationResponse;
import org.apache.flink.runtime.registration.RetryingRegistration;
import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
+import org.apache.flink.runtime.concurrent.Future;
import org.slf4j.Logger;
@@ -36,115 +34,46 @@ import java.util.UUID;
import java.util.concurrent.Executor;
import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.Preconditions.checkState;
/**
* The connection between a TaskExecutor and the ResourceManager.
*/
-public class TaskExecutorToResourceManagerConnection {
-
- /** the logger for all log messages of this class */
- private final Logger log;
+public class TaskExecutorToResourceManagerConnection
+ extends RegisteredRpcConnection<ResourceManagerGateway, TaskExecutorRegistrationSuccess> {
/** the TaskExecutor whose connection to the ResourceManager this represents */
private final TaskExecutor taskExecutor;
- private final UUID resourceManagerLeaderId;
-
- private final String resourceManagerAddress;
-
- /** Execution context to be used to execute the on complete action of the ResourceManagerRegistration */
- private final Executor executor;
-
- private TaskExecutorToResourceManagerConnection.ResourceManagerRegistration pendingRegistration;
-
- private volatile ResourceManagerGateway registeredResourceManager;
-
private InstanceID registrationId;
- /** flag indicating that the connection is closed */
- private volatile boolean closed;
-
-
public TaskExecutorToResourceManagerConnection(
- Logger log,
- TaskExecutor taskExecutor,
- String resourceManagerAddress,
- UUID resourceManagerLeaderId,
- Executor executor) {
+ Logger log,
+ TaskExecutor taskExecutor,
+ String resourceManagerAddress,
+ UUID resourceManagerLeaderId,
+ Executor executor) {
- this.log = checkNotNull(log);
+ super(log, resourceManagerAddress, resourceManagerLeaderId, executor);
this.taskExecutor = checkNotNull(taskExecutor);
- this.resourceManagerAddress = checkNotNull(resourceManagerAddress);
- this.resourceManagerLeaderId = checkNotNull(resourceManagerLeaderId);
- this.executor = checkNotNull(executor);
- }
-
- // ------------------------------------------------------------------------
- // Life cycle
- // ------------------------------------------------------------------------
-
- @SuppressWarnings("unchecked")
- public void start() {
- checkState(!closed, "The connection is already closed");
- checkState(!isRegistered() && pendingRegistration == null, "The connection is already started");
-
- pendingRegistration = new TaskExecutorToResourceManagerConnection.ResourceManagerRegistration(
- log, taskExecutor.getRpcService(),
- resourceManagerAddress, resourceManagerLeaderId,
- taskExecutor.getAddress(), taskExecutor.getResourceID());
- pendingRegistration.startRegistration();
-
- Future<Tuple2<ResourceManagerGateway, TaskExecutorRegistrationSuccess>> future = pendingRegistration.getFuture();
-
- future.thenAcceptAsync(new AcceptFunction<Tuple2<ResourceManagerGateway, TaskExecutorRegistrationSuccess>>() {
- @Override
- public void accept(Tuple2<ResourceManagerGateway, TaskExecutorRegistrationSuccess> result) {
- registrationId = result.f1.getRegistrationId();
- registeredResourceManager = result.f0;
- }
- }, executor);
-
- // this future should only ever fail if there is a bug, not if the registration is declined
- future.exceptionallyAsync(new ApplyFunction<Throwable, Void>() {
- @Override
- public Void apply(Throwable failure) {
- taskExecutor.onFatalErrorAsync(failure);
- return null;
- }
- }, executor);
- }
-
- public void close() {
- closed = true;
-
- // make sure we do not keep re-trying forever
- if (pendingRegistration != null) {
- pendingRegistration.cancel();
- }
}
- public boolean isClosed() {
- return closed;
- }
- // ------------------------------------------------------------------------
- // Properties
- // ------------------------------------------------------------------------
-
- public UUID getResourceManagerLeaderId() {
- return resourceManagerLeaderId;
+ @Override
+ protected RetryingRegistration<ResourceManagerGateway, TaskExecutorRegistrationSuccess> generateRegistration() {
+ return new TaskExecutorToResourceManagerConnection.ResourceManagerRegistration(
+ log, taskExecutor.getRpcService(),
+ getTargetAddress(), getTargetLeaderId(),
+ taskExecutor.getAddress(),taskExecutor.getResourceID());
}
- public String getResourceManagerAddress() {
- return resourceManagerAddress;
+ @Override
+ protected void onRegistrationSuccess(TaskExecutorRegistrationSuccess success) {
+ registrationId = success.getRegistrationId();
}
- /**
- * Gets the ResourceManagerGateway. This returns null until the registration is completed.
- */
- public ResourceManagerGateway getResourceManager() {
- return registeredResourceManager;
+ @Override
+ protected void onRegistrationFailure(Throwable failure) {
+ taskExecutor.onFatalErrorAsync(failure);
}
/**
@@ -155,18 +84,6 @@ public class TaskExecutorToResourceManagerConnection {
return registrationId;
}
- public boolean isRegistered() {
- return registeredResourceManager != null;
- }
-
- // ------------------------------------------------------------------------
-
- @Override
- public String toString() {
- return String.format("Connection to ResourceManager %s (leaderId=%s)",
- resourceManagerAddress, resourceManagerLeaderId);
- }
-
// ------------------------------------------------------------------------
// Utilities
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/20e4e12d/flink-runtime/src/test/java/org/apache/flink/runtime/registration/RegisteredRpcConnectionTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/registration/RegisteredRpcConnectionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/registration/RegisteredRpcConnectionTest.java
new file mode 100644
index 0000000..8558205
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/registration/RegisteredRpcConnectionTest.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.registration;
+
+import org.apache.flink.runtime.registration.RetryingRegistrationTest.TestRegistrationSuccess;
+import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.runtime.rpc.TestingRpcService;
+import org.apache.flink.util.TestLogger;
+import org.junit.Test;
+import org.slf4j.LoggerFactory;
+
+import java.util.UUID;
+import java.util.concurrent.Executor;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for RegisteredRpcConnection, validating the successful, failure and close behavior.
+ */
+public class RegisteredRpcConnectionTest extends TestLogger {
+
+ @Test
+ public void testSuccessfulRpcConnection() throws Exception {
+ final String testRpcConnectionEndpointAddress = "<TestRpcConnectionEndpointAddress>";
+ final UUID leaderId = UUID.randomUUID();
+ final String connectionID = "Test RPC Connection ID";
+
+ // an endpoint that immediately returns success
+ TestRegistrationGateway testGateway = new TestRegistrationGateway(new RetryingRegistrationTest.TestRegistrationSuccess(connectionID));
+ TestingRpcService rpcService = new TestingRpcService();
+
+ try {
+ rpcService.registerGateway(testRpcConnectionEndpointAddress, testGateway);
+
+ TestRpcConnection connection = new TestRpcConnection(testRpcConnectionEndpointAddress, leaderId, rpcService.getExecutor(), rpcService);
+ connection.start();
+
+ //wait for connection established
+ Thread.sleep(RetryingRegistrationTest.TestRetryingRegistration.MAX_TIMEOUT);
+
+ // validate correct invocation and result
+ assertTrue(connection.isConnected());
+ assertEquals(testRpcConnectionEndpointAddress, connection.getTargetAddress());
+ assertEquals(leaderId, connection.getTargetLeaderId());
+ assertEquals(testGateway, connection.getTargetGateway());
+ assertEquals(connectionID, connection.getConnectionId());
+ }
+ finally {
+ testGateway.stop();
+ rpcService.stopService();
+ }
+ }
+
+ @Test
+ public void testRpcConnectionFailures() throws Exception {
+ final String connectionFailureMessage = "Test RPC Connection failure";
+ final String testRpcConnectionEndpointAddress = "<TestRpcConnectionEndpointAddress>";
+ final UUID leaderId = UUID.randomUUID();
+
+ TestingRpcService rpcService = new TestingRpcService();
+
+ try {
+ // gateway that upon calls Throw an exception
+ TestRegistrationGateway testGateway = mock(TestRegistrationGateway.class);
+ when(testGateway.registrationCall(any(UUID.class), anyLong())).thenThrow(
+ new RuntimeException(connectionFailureMessage));
+
+ rpcService.registerGateway(testRpcConnectionEndpointAddress, testGateway);
+
+ TestRpcConnection connection = new TestRpcConnection(testRpcConnectionEndpointAddress, leaderId, rpcService.getExecutor(), rpcService);
+ connection.start();
+
+ //wait for connection failure
+ Thread.sleep(RetryingRegistrationTest.TestRetryingRegistration.MAX_TIMEOUT);
+
+ // validate correct invocation and result
+ assertFalse(connection.isConnected());
+ assertEquals(testRpcConnectionEndpointAddress, connection.getTargetAddress());
+ assertEquals(leaderId, connection.getTargetLeaderId());
+ assertNull(connection.getTargetGateway());
+ assertEquals(connectionFailureMessage, connection.getFailareMessage());
+ }
+ finally {
+ rpcService.stopService();
+ }
+ }
+
+ @Test
+ public void testRpcConnectionClose() throws Exception {
+ final String testRpcConnectionEndpointAddress = "<TestRpcConnectionEndpointAddress>";
+ final UUID leaderId = UUID.randomUUID();
+ final String connectionID = "Test RPC Connection ID";
+
+ TestRegistrationGateway testGateway = new TestRegistrationGateway(new RetryingRegistrationTest.TestRegistrationSuccess(connectionID));
+ TestingRpcService rpcService = new TestingRpcService();
+
+ try{
+ rpcService.registerGateway(testRpcConnectionEndpointAddress, testGateway);
+
+ TestRpcConnection connection = new TestRpcConnection(testRpcConnectionEndpointAddress, leaderId, rpcService.getExecutor(), rpcService);
+ connection.start();
+ //close the connection
+ connection.close();
+
+ // validate connection is closed
+ assertEquals(testRpcConnectionEndpointAddress, connection.getTargetAddress());
+ assertEquals(leaderId, connection.getTargetLeaderId());
+ assertTrue(connection.isClosed());
+ }
+ finally {
+ testGateway.stop();
+ rpcService.stopService();
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // test RegisteredRpcConnection
+ // ------------------------------------------------------------------------
+
+ private static class TestRpcConnection extends RegisteredRpcConnection<TestRegistrationGateway, TestRegistrationSuccess> {
+
+ private final RpcService rpcService;
+
+ private String connectionId;
+
+ private String failureMessage;
+
+ public TestRpcConnection(String targetAddress,
+ UUID targetLeaderId,
+ Executor executor,
+ RpcService rpcService)
+ {
+ super(LoggerFactory.getLogger(RegisteredRpcConnectionTest.class), targetAddress, targetLeaderId, executor);
+ this.rpcService = rpcService;
+ }
+
+ @Override
+ protected RetryingRegistration<TestRegistrationGateway, RetryingRegistrationTest.TestRegistrationSuccess> generateRegistration() {
+ return new RetryingRegistrationTest.TestRetryingRegistration(rpcService, getTargetAddress(), getTargetLeaderId());
+ }
+
+ @Override
+ protected void onRegistrationSuccess(RetryingRegistrationTest.TestRegistrationSuccess success) {
+ connectionId = success.getCorrelationId();
+ }
+
+ @Override
+ protected void onRegistrationFailure(Throwable failure) {
+ failureMessage = failure.getMessage();
+ }
+
+ public String getConnectionId() {
+ return connectionId;
+ }
+
+ public String getFailareMessage() {
+ return failureMessage;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/20e4e12d/flink-runtime/src/test/java/org/apache/flink/runtime/registration/RetryingRegistrationTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/registration/RetryingRegistrationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/registration/RetryingRegistrationTest.java
index e56a9ec..6d6bbef 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/registration/RetryingRegistrationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/registration/RetryingRegistrationTest.java
@@ -298,12 +298,12 @@ public class RetryingRegistrationTest extends TestLogger {
// test registration
// ------------------------------------------------------------------------
- private static class TestRegistrationSuccess extends RegistrationResponse.Success {
+ protected static class TestRegistrationSuccess extends RegistrationResponse.Success {
private static final long serialVersionUID = 5542698790917150604L;
private final String correlationId;
- private TestRegistrationSuccess(String correlationId) {
+ public TestRegistrationSuccess(String correlationId) {
this.correlationId = correlationId;
}
@@ -312,7 +312,7 @@ public class RetryingRegistrationTest extends TestLogger {
}
}
- private static class TestRetryingRegistration extends RetryingRegistration<TestRegistrationGateway, TestRegistrationSuccess> {
+ protected static class TestRetryingRegistration extends RetryingRegistration<TestRegistrationGateway, TestRegistrationSuccess> {
// we use shorter timeouts here to speed up the tests
static final long INITIAL_TIMEOUT = 20;
[20/50] [abbrv] flink git commit: [FLINK-4746] Make
TaskManagerRuntimeInfo an interface
Posted by tr...@apache.org.
[FLINK-4746] Make TaskManagerRuntimeInfo an interface
Let the TaskManagerConfiguration implement the TaskManagerRuntimeInformation to make some of
the TaskManager's configuration values accessible from different components.
This closes #2599.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/205de723
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/205de723
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/205de723
Branch: refs/heads/flip-6
Commit: 205de723f781f265b285e72adb0b79f5d348951c
Parents: 0655adc
Author: Till Rohrmann <tr...@apache.org>
Authored: Wed Oct 5 14:47:24 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:30 2016 +0100
----------------------------------------------------------------------
.../runtime/taskexecutor/TaskExecutor.java | 11 +---
.../taskexecutor/TaskManagerConfiguration.java | 22 +++----
.../taskmanager/TaskManagerRuntimeInfo.java | 61 ++------------------
.../flink/runtime/taskmanager/TaskManager.scala | 11 +---
.../operators/drivers/TestTaskContext.java | 4 +-
.../testutils/BinaryOperatorTestBase.java | 4 +-
.../operators/testutils/DriverTestBase.java | 4 +-
.../operators/testutils/MockEnvironment.java | 8 +--
.../testutils/UnaryOperatorTestBase.java | 4 +-
.../runtime/taskexecutor/TaskExecutorTest.java | 8 +--
.../runtime/taskmanager/TaskAsyncCallTest.java | 3 +-
.../flink/runtime/taskmanager/TaskTest.java | 3 +-
.../util/TestingTaskManagerRuntimeInfo.java | 52 +++++++++++++++++
.../tasks/InterruptSensitiveRestoreTest.java | 5 +-
.../runtime/tasks/StreamMockEnvironment.java | 4 +-
.../streaming/runtime/tasks/StreamTaskTest.java | 4 +-
16 files changed, 97 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index 35b639b..a2716e5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.taskexecutor;
-import org.apache.flink.configuration.UnmodifiableConfiguration;
import org.apache.flink.runtime.accumulators.AccumulatorSnapshot;
import org.apache.flink.runtime.blob.BlobCache;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
@@ -71,7 +70,6 @@ import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
import org.apache.flink.util.Preconditions;
import java.util.HashSet;
@@ -127,9 +125,6 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
private final FileCache fileCache;
- // TODO: Try to get rid of it
- private final TaskManagerRuntimeInfo taskManagerRuntimeInfo;
-
// --------- resource manager --------
private TaskExecutorToResourceManagerConnection resourceManagerConnection;
@@ -177,10 +172,6 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
this.taskManagerMetricGroup = checkNotNull(taskManagerMetricGroup);
this.broadcastVariableManager = checkNotNull(broadcastVariableManager);
this.fileCache = checkNotNull(fileCache);
- this.taskManagerRuntimeInfo = new TaskManagerRuntimeInfo(
- taskManagerLocation.getHostname(),
- new UnmodifiableConfiguration(taskManagerConfiguration.getConfiguration()),
- taskManagerConfiguration.getTmpDirPaths());
this.jobManagerConnections = new HashMap<>(4);
@@ -308,7 +299,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
checkpointResponder,
libraryCache,
fileCache,
- taskManagerRuntimeInfo,
+ taskManagerConfiguration,
taskMetricGroup,
resultPartitionConsumableNotifier,
partitionStateChecker,
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
index bce3dc3..1d1e732 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java
@@ -23,6 +23,7 @@ import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.UnmodifiableConfiguration;
import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,13 +34,13 @@ import java.io.File;
/**
* Configuration object for {@link TaskExecutor}.
*/
-public class TaskManagerConfiguration {
+public class TaskManagerConfiguration implements TaskManagerRuntimeInfo {
private static final Logger LOG = LoggerFactory.getLogger(TaskManagerConfiguration.class);
private final int numberSlots;
- private final String[] tmpDirPaths;
+ private final String[] tmpDirectories;
private final Time timeout;
// null indicates an infinite duration
@@ -50,12 +51,11 @@ public class TaskManagerConfiguration {
private final long cleanupInterval;
- // TODO: remove necessity for complete configuration object
- private final Configuration configuration;
+ private final UnmodifiableConfiguration configuration;
public TaskManagerConfiguration(
int numberSlots,
- String[] tmpDirPaths,
+ String[] tmpDirectories,
Time timeout,
Time maxRegistrationDuration,
Time initialRegistrationPause,
@@ -65,7 +65,7 @@ public class TaskManagerConfiguration {
Configuration configuration) {
this.numberSlots = numberSlots;
- this.tmpDirPaths = Preconditions.checkNotNull(tmpDirPaths);
+ this.tmpDirectories = Preconditions.checkNotNull(tmpDirectories);
this.timeout = Preconditions.checkNotNull(timeout);
this.maxRegistrationDuration = maxRegistrationDuration;
this.initialRegistrationPause = Preconditions.checkNotNull(initialRegistrationPause);
@@ -79,10 +79,6 @@ public class TaskManagerConfiguration {
return numberSlots;
}
- public String[] getTmpDirPaths() {
- return tmpDirPaths;
- }
-
public Time getTimeout() {
return timeout;
}
@@ -107,10 +103,16 @@ public class TaskManagerConfiguration {
return cleanupInterval;
}
+ @Override
public Configuration getConfiguration() {
return configuration;
}
+ @Override
+ public String[] getTmpDirectories() {
+ return tmpDirectories;
+ }
+
// --------------------------------------------------------------------------------------------
// Static factory methods
// --------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerRuntimeInfo.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerRuntimeInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerRuntimeInfo.java
index 9ac982e..d1efe34 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerRuntimeInfo.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManagerRuntimeInfo.java
@@ -20,71 +20,22 @@ package org.apache.flink.runtime.taskmanager;
import org.apache.flink.configuration.Configuration;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.Preconditions.checkArgument;
-
/**
- * Encapsulation of TaskManager runtime information, like hostname and configuration.
+ * Interface to access {@link TaskManager} information.
*/
-public class TaskManagerRuntimeInfo implements java.io.Serializable {
-
- private static final long serialVersionUID = 5598219619760274072L;
-
- /** host name of the interface that the TaskManager uses to communicate */
- private final String hostname;
-
- /** configuration that the TaskManager was started with */
- private final Configuration configuration;
-
- /** list of temporary file directories */
- private final String[] tmpDirectories;
-
- /**
- * Creates a runtime info.
- *
- * @param hostname The host name of the interface that the TaskManager uses to communicate.
- * @param configuration The configuration that the TaskManager was started with.
- * @param tmpDirectory The temporary file directory.
- */
- public TaskManagerRuntimeInfo(String hostname, Configuration configuration, String tmpDirectory) {
- this(hostname, configuration, new String[] { tmpDirectory });
- }
-
- /**
- * Creates a runtime info.
- * @param hostname The host name of the interface that the TaskManager uses to communicate.
- * @param configuration The configuration that the TaskManager was started with.
- * @param tmpDirectories The list of temporary file directories.
- */
- public TaskManagerRuntimeInfo(String hostname, Configuration configuration, String[] tmpDirectories) {
- checkArgument(tmpDirectories.length > 0);
- this.hostname = checkNotNull(hostname);
- this.configuration = checkNotNull(configuration);
- this.tmpDirectories = tmpDirectories;
-
- }
-
- /**
- * Gets host name of the interface that the TaskManager uses to communicate.
- * @return The host name of the interface that the TaskManager uses to communicate.
- */
- public String getHostname() {
- return hostname;
- }
+public interface TaskManagerRuntimeInfo {
/**
* Gets the configuration that the TaskManager was started with.
+ *
* @return The configuration that the TaskManager was started with.
*/
- public Configuration getConfiguration() {
- return configuration;
- }
+ Configuration getConfiguration();
/**
* Gets the list of temporary file directories.
+ *
* @return The list of temporary file directories.
*/
- public String[] getTmpDirectories() {
- return tmpDirectories;
- }
+ String[] getTmpDirectories();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 3ed8ea6..5e0510d 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -145,7 +145,7 @@ class TaskManager(
protected val bcVarManager = new BroadcastVariableManager()
/** Handler for distributed files cached by this TaskManager */
- protected val fileCache = new FileCache(config.getTmpDirPaths())
+ protected val fileCache = new FileCache(config.getTmpDirectories())
private var taskManagerMetricGroup : TaskManagerMetricGroup = _
@@ -167,11 +167,6 @@ class TaskManager(
var leaderSessionID: Option[UUID] = None
- private val runtimeInfo = new TaskManagerRuntimeInfo(
- location.getHostname(),
- new UnmodifiableConfiguration(config.getConfiguration()),
- config.getTmpDirPaths())
-
private var scheduledTaskManagerRegistration: Option[Cancellable] = None
private var currentRegistrationRun: UUID = UUID.randomUUID()
@@ -972,7 +967,7 @@ class TaskManager(
}
taskManagerMetricGroup =
- new TaskManagerMetricGroup(metricsRegistry, this.runtimeInfo.getHostname, id.toString)
+ new TaskManagerMetricGroup(metricsRegistry, location.getHostname, id.toString)
MetricUtils.instantiateStatusMetrics(taskManagerMetricGroup)
MetricUtils.instantiateNetworkMetrics(taskManagerMetricGroup, network)
@@ -1155,7 +1150,7 @@ class TaskManager(
checkpointResponder,
libCache,
fileCache,
- runtimeInfo,
+ config,
taskMetricGroup,
resultPartitionConsumableNotifier,
partitionStateChecker,
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/TestTaskContext.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/TestTaskContext.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/TestTaskContext.java
index 5acc915..4fa74b3 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/TestTaskContext.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/TestTaskContext.java
@@ -35,6 +35,7 @@ import org.apache.flink.runtime.operators.testutils.DummyInvokable;
import org.apache.flink.runtime.operators.testutils.UnregisteredTaskMetricsGroup;
import org.apache.flink.runtime.operators.util.TaskConfig;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import org.apache.flink.util.Collector;
import org.apache.flink.util.MutableObjectIterator;
@@ -74,8 +75,7 @@ public class TestTaskContext<S, T> implements TaskContext<S, T> {
public TestTaskContext(long memoryInBytes) {
this.memoryManager = new MemoryManager(memoryInBytes, 1, 32 * 1024, MemoryType.HEAP, true);
- this.taskManageInfo = new TaskManagerRuntimeInfo(
- "localhost", new Configuration(), System.getProperty("java.io.tmpdir"));
+ this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
// --------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/BinaryOperatorTestBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/BinaryOperatorTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/BinaryOperatorTestBase.java
index 47a7953..790ac9e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/BinaryOperatorTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/BinaryOperatorTestBase.java
@@ -39,6 +39,7 @@ import org.apache.flink.runtime.operators.TaskContext;
import org.apache.flink.runtime.operators.sort.UnilateralSortMerger;
import org.apache.flink.runtime.operators.util.TaskConfig;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import org.apache.flink.util.Collector;
import org.apache.flink.util.MutableObjectIterator;
import org.apache.flink.util.TestLogger;
@@ -111,8 +112,7 @@ public class BinaryOperatorTestBase<S extends Function, IN, OUT> extends TestLog
this.owner = new DummyInvokable();
this.taskConfig = new TaskConfig(new Configuration());
this.executionConfig = executionConfig;
- this.taskManageInfo = new TaskManagerRuntimeInfo(
- "localhost", new Configuration(), System.getProperty("java.io.tmpdir"));
+ this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
@Parameterized.Parameters
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
index c9fa664..6f132c9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
@@ -37,6 +37,7 @@ import org.apache.flink.runtime.operators.util.TaskConfig;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
import org.apache.flink.runtime.testutils.recordutils.RecordComparator;
import org.apache.flink.runtime.testutils.recordutils.RecordSerializerFactory;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
import org.apache.flink.util.MutableObjectIterator;
@@ -114,8 +115,7 @@ public class DriverTestBase<S extends Function> extends TestLogger implements Ta
this.owner = new DummyInvokable();
this.taskConfig = new TaskConfig(new Configuration());
this.executionConfig = executionConfig;
- this.taskManageInfo = new TaskManagerRuntimeInfo(
- "localhost", new Configuration(), System.getProperty("java.io.tmpdir"));
+ this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
@Parameterized.Parameters
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
index 08b84cb..9b33071 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
@@ -22,7 +22,6 @@ import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.TaskInfo;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.UnmodifiableConfiguration;
import org.apache.flink.core.fs.Path;
import org.apache.flink.core.memory.MemorySegmentFactory;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
@@ -48,8 +47,10 @@ import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.query.KvStateRegistry;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import org.apache.flink.types.Record;
import org.apache.flink.util.MutableObjectIterator;
+
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -233,10 +234,7 @@ public class MockEnvironment implements Environment {
@Override
public TaskManagerRuntimeInfo getTaskManagerInfo() {
- return new TaskManagerRuntimeInfo(
- "localhost",
- new UnmodifiableConfiguration(new Configuration()),
- System.getProperty("java.io.tmpdir"));
+ return new TestingTaskManagerRuntimeInfo();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/UnaryOperatorTestBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/UnaryOperatorTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/UnaryOperatorTestBase.java
index 19744e1..af8a29d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/UnaryOperatorTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/UnaryOperatorTestBase.java
@@ -37,6 +37,7 @@ import org.apache.flink.runtime.operators.ResettableDriver;
import org.apache.flink.runtime.operators.sort.UnilateralSortMerger;
import org.apache.flink.runtime.operators.util.TaskConfig;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import org.apache.flink.util.Collector;
import org.apache.flink.util.MutableObjectIterator;
@@ -115,8 +116,7 @@ public class UnaryOperatorTestBase<S extends Function, IN, OUT> extends TestLogg
this.executionConfig = executionConfig;
this.comparators = new ArrayList<TypeComparator<IN>>(2);
- this.taskManageInfo = new TaskManagerRuntimeInfo(
- "localhost", new Configuration(), System.getProperty("java.io.tmpdir"));
+ this.taskManageInfo = new TestingTaskManagerRuntimeInfo();
}
@Parameterized.Parameters
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index f5fe52c..ecbd9b5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -63,14 +63,11 @@ public class TaskExecutorTest extends TestLogger {
ResourceManagerGateway rmGateway = mock(ResourceManagerGateway.class);
TaskManagerConfiguration taskManagerServicesConfiguration = mock(TaskManagerConfiguration.class);
PowerMockito.when(taskManagerServicesConfiguration.getNumberSlots()).thenReturn(1);
- PowerMockito.when(taskManagerServicesConfiguration.getConfiguration()).thenReturn(new Configuration());
- PowerMockito.when(taskManagerServicesConfiguration.getTmpDirPaths()).thenReturn(new String[1]);
rpc.registerGateway(resourceManagerAddress, rmGateway);
TaskManagerLocation taskManagerLocation = mock(TaskManagerLocation.class);
when(taskManagerLocation.getResourceID()).thenReturn(resourceID);
- when(taskManagerLocation.getHostname()).thenReturn("foobar");
NonHaServices haServices = new NonHaServices(resourceManagerAddress);
@@ -124,7 +121,7 @@ public class TaskExecutorTest extends TestLogger {
TaskManagerConfiguration taskManagerServicesConfiguration = mock(TaskManagerConfiguration.class);
PowerMockito.when(taskManagerServicesConfiguration.getNumberSlots()).thenReturn(1);
PowerMockito.when(taskManagerServicesConfiguration.getConfiguration()).thenReturn(new Configuration());
- PowerMockito.when(taskManagerServicesConfiguration.getTmpDirPaths()).thenReturn(new String[1]);
+ PowerMockito.when(taskManagerServicesConfiguration.getTmpDirectories()).thenReturn(new String[1]);
TaskManagerLocation taskManagerLocation = mock(TaskManagerLocation.class);
when(taskManagerLocation.getResourceID()).thenReturn(resourceID);
@@ -198,12 +195,9 @@ public class TaskExecutorTest extends TestLogger {
TaskManagerConfiguration taskManagerServicesConfiguration = mock(TaskManagerConfiguration.class);
PowerMockito.when(taskManagerServicesConfiguration.getNumberSlots()).thenReturn(1);
- PowerMockito.when(taskManagerServicesConfiguration.getConfiguration()).thenReturn(new Configuration());
- PowerMockito.when(taskManagerServicesConfiguration.getTmpDirPaths()).thenReturn(new String[1]);
TaskManagerLocation taskManagerLocation = mock(TaskManagerLocation.class);
when(taskManagerLocation.getResourceID()).thenReturn(resourceID);
- when(taskManagerLocation.getHostname()).thenReturn("foobar");
TaskExecutor taskManager = new TaskExecutor(
taskManagerServicesConfiguration,
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
index d994134..fefc00b80 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
@@ -45,6 +45,7 @@ import org.apache.flink.runtime.memory.MemoryManager;
import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
import org.apache.flink.runtime.state.TaskStateHandles;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import org.apache.flink.util.SerializedValue;
import org.junit.Before;
@@ -174,7 +175,7 @@ public class TaskAsyncCallTest {
mock(CheckpointResponder.class),
libCache,
mock(FileCache.class),
- new TaskManagerRuntimeInfo("localhost", new Configuration(), System.getProperty("java.io.tmpdir")),
+ new TestingTaskManagerRuntimeInfo(),
mock(TaskMetricGroup.class),
consumableNotifier,
partitionStateChecker,
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
index f28e58a..1d2f9ed 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
@@ -50,6 +50,7 @@ import org.apache.flink.runtime.messages.TaskManagerMessages;
import org.apache.flink.runtime.messages.TaskMessages;
import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import org.apache.flink.util.SerializedValue;
import org.apache.flink.util.TestLogger;
import org.junit.After;
@@ -802,7 +803,7 @@ public class TaskTest extends TestLogger {
checkpointResponder,
libCache,
mock(FileCache.class),
- new TaskManagerRuntimeInfo("localhost", new Configuration(), System.getProperty("java.io.tmpdir")),
+ new TestingTaskManagerRuntimeInfo(),
mock(TaskMetricGroup.class),
consumableNotifier,
partitionStateChecker,
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-runtime/src/test/java/org/apache/flink/runtime/util/TestingTaskManagerRuntimeInfo.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/util/TestingTaskManagerRuntimeInfo.java b/flink-runtime/src/test/java/org/apache/flink/runtime/util/TestingTaskManagerRuntimeInfo.java
new file mode 100644
index 0000000..e56da97
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/util/TestingTaskManagerRuntimeInfo.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.util;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+
+import java.io.File;
+
+/**
+ * TaskManagerRuntimeInfo implementation for testing purposes
+ */
+public class TestingTaskManagerRuntimeInfo implements TaskManagerRuntimeInfo {
+
+ private final Configuration configuration;
+ private final String[] tmpDirectories;
+
+ public TestingTaskManagerRuntimeInfo() {
+ this(new Configuration(), System.getProperty("java.io.tmpdir").split(",|" + File.pathSeparator));
+ }
+
+ public TestingTaskManagerRuntimeInfo(Configuration configuration, String[] tmpDirectories) {
+ this.configuration = configuration;
+ this.tmpDirectories = tmpDirectories;
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ @Override
+ public String[] getTmpDirectories() {
+ return tmpDirectories;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java
index 861665f..1077052 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/InterruptSensitiveRestoreTest.java
@@ -49,8 +49,8 @@ import org.apache.flink.runtime.state.TaskStateHandles;
import org.apache.flink.runtime.taskmanager.CheckpointResponder;
import org.apache.flink.runtime.taskmanager.Task;
import org.apache.flink.runtime.taskmanager.TaskManagerActions;
-import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
import org.apache.flink.runtime.util.EnvironmentInformation;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.checkpoint.Checkpointed;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
@@ -175,8 +175,7 @@ public class InterruptSensitiveRestoreTest {
mock(CheckpointResponder.class),
new FallbackLibraryCacheManager(),
new FileCache(tmpDirectories),
- new TaskManagerRuntimeInfo(
- "localhost", new Configuration(), EnvironmentInformation.getTemporaryFileDirectory()),
+ new TestingTaskManagerRuntimeInfo(new Configuration(), tmpDirectories),
new UnregisteredTaskMetricsGroup(),
mock(ResultPartitionConsumableNotifier.class),
mock(PartitionStateChecker.class),
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java
index 2376a60..d51fb33 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java
@@ -52,6 +52,8 @@ import org.apache.flink.runtime.plugable.NonReusingDeserializationDelegate;
import org.apache.flink.runtime.query.KvStateRegistry;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
+
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -332,7 +334,7 @@ public class StreamMockEnvironment implements Environment {
@Override
public TaskManagerRuntimeInfo getTaskManagerInfo() {
- return new TaskManagerRuntimeInfo("localhost", new Configuration(), System.getProperty("java.io.tmpdir"));
+ return new TestingTaskManagerRuntimeInfo();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/205de723/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
index ee0839f..bb246f9 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
@@ -52,7 +52,7 @@ import org.apache.flink.runtime.taskmanager.Task;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import org.apache.flink.runtime.taskmanager.TaskExecutionStateListener;
import org.apache.flink.runtime.taskmanager.TaskManagerActions;
-import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.api.graph.StreamConfig;
@@ -243,7 +243,7 @@ public class StreamTaskTest {
mock(CheckpointResponder.class),
libCache,
mock(FileCache.class),
- new TaskManagerRuntimeInfo("localhost", taskManagerConfig, System.getProperty("java.io.tmpdir")),
+ new TestingTaskManagerRuntimeInfo(taskManagerConfig, new String[] {System.getProperty("java.io.tmpdir")}),
new UnregisteredTaskMetricsGroup(),
consumableNotifier,
partitionStateChecker,
[16/50] [abbrv] flink git commit: [FLINK-4347][FLINK-4348] simplify
SlotManager and integrate it with ResourceManager
Posted by tr...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
index 8f09152..14afd0e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
@@ -25,10 +25,8 @@ import org.apache.flink.runtime.jobmaster.JobMasterGateway;
import org.apache.flink.runtime.jobmaster.JobMasterRegistrationSuccess;
import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SimpleSlotManager;
import org.apache.flink.runtime.rpc.TestingSerialRpcService;
import org.apache.flink.runtime.registration.RegistrationResponse;
-import org.apache.flink.runtime.rpc.exceptions.LeaderSessionIDException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -160,7 +158,7 @@ public class ResourceManagerJobMasterTest {
TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
highAvailabilityServices.setResourceManagerLeaderElectionService(resourceManagerLeaderElectionService);
highAvailabilityServices.setJobMasterLeaderRetriever(jobID, jobMasterLeaderRetrievalService);
- ResourceManager resourceManager = new StandaloneResourceManager(rpcService, highAvailabilityServices, new SimpleSlotManager());
+ ResourceManager resourceManager = new TestingResourceManager(rpcService, highAvailabilityServices);
resourceManager.start();
return resourceManager;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
index e6d1ed5..a577c26 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
@@ -22,10 +22,9 @@ import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SimpleSlotManager;
import org.apache.flink.runtime.rpc.TestingSerialRpcService;
import org.apache.flink.runtime.registration.RegistrationResponse;
-import org.apache.flink.runtime.rpc.exceptions.LeaderSessionIDException;
+import org.apache.flink.runtime.taskexecutor.SlotReport;
import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
import org.apache.flink.runtime.taskexecutor.TaskExecutorRegistrationSuccess;
import org.junit.After;
@@ -44,9 +43,24 @@ public class ResourceManagerTaskExecutorTest {
private TestingSerialRpcService rpcService;
+ private SlotReport slotReport = new SlotReport();
+
+ private static String taskExecutorAddress = "/taskExecutor1";
+
+ private ResourceID taskExecutorResourceID;
+
+ private StandaloneResourceManager resourceManager;
+
+ private UUID leaderSessionId;
+
@Before
public void setup() throws Exception {
rpcService = new TestingSerialRpcService();
+
+ taskExecutorResourceID = mockTaskExecutor(taskExecutorAddress);
+ TestingLeaderElectionService rmLeaderElectionService = new TestingLeaderElectionService();
+ resourceManager = createAndStartResourceManager(rmLeaderElectionService);
+ leaderSessionId = grantLeadership(rmLeaderElectionService);
}
@After
@@ -59,19 +73,15 @@ public class ResourceManagerTaskExecutorTest {
*/
@Test
public void testRegisterTaskExecutor() throws Exception {
- String taskExecutorAddress = "/taskExecutor1";
- ResourceID taskExecutorResourceID = mockTaskExecutor(taskExecutorAddress);
- TestingLeaderElectionService rmLeaderElectionService = new TestingLeaderElectionService();
- final ResourceManager resourceManager = createAndStartResourceManager(rmLeaderElectionService);
- final UUID leaderSessionId = grantLeadership(rmLeaderElectionService);
-
// test response successful
- Future<RegistrationResponse> successfulFuture = resourceManager.registerTaskExecutor(leaderSessionId, taskExecutorAddress, taskExecutorResourceID);
+ Future<RegistrationResponse> successfulFuture =
+ resourceManager.registerTaskExecutor(leaderSessionId, taskExecutorAddress, taskExecutorResourceID, slotReport);
RegistrationResponse response = successfulFuture.get(5, TimeUnit.SECONDS);
assertTrue(response instanceof TaskExecutorRegistrationSuccess);
// test response successful with instanceID not equal to previous when receive duplicate registration from taskExecutor
- Future<RegistrationResponse> duplicateFuture = resourceManager.registerTaskExecutor(leaderSessionId, taskExecutorAddress, taskExecutorResourceID);
+ Future<RegistrationResponse> duplicateFuture =
+ resourceManager.registerTaskExecutor(leaderSessionId, taskExecutorAddress, taskExecutorResourceID, slotReport);
RegistrationResponse duplicateResponse = duplicateFuture.get();
assertTrue(duplicateResponse instanceof TaskExecutorRegistrationSuccess);
assertNotEquals(((TaskExecutorRegistrationSuccess) response).getRegistrationId(), ((TaskExecutorRegistrationSuccess) duplicateResponse).getRegistrationId());
@@ -82,15 +92,10 @@ public class ResourceManagerTaskExecutorTest {
*/
@Test
public void testRegisterTaskExecutorWithUnmatchedLeaderSessionId() throws Exception {
- String taskExecutorAddress = "/taskExecutor1";
- ResourceID taskExecutorResourceID = mockTaskExecutor(taskExecutorAddress);
- TestingLeaderElectionService rmLeaderElectionService = new TestingLeaderElectionService();
- final ResourceManager resourceManager = createAndStartResourceManager(rmLeaderElectionService);
- final UUID leaderSessionId = grantLeadership(rmLeaderElectionService);
-
// test throw exception when receive a registration from taskExecutor which takes unmatched leaderSessionId
UUID differentLeaderSessionID = UUID.randomUUID();
- Future<RegistrationResponse> unMatchedLeaderFuture = resourceManager.registerTaskExecutor(differentLeaderSessionID, taskExecutorAddress, taskExecutorResourceID);
+ Future<RegistrationResponse> unMatchedLeaderFuture =
+ resourceManager.registerTaskExecutor(differentLeaderSessionID, taskExecutorAddress, taskExecutorResourceID, slotReport);
assertTrue(unMatchedLeaderFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
}
@@ -99,15 +104,10 @@ public class ResourceManagerTaskExecutorTest {
*/
@Test
public void testRegisterTaskExecutorFromInvalidAddress() throws Exception {
- String taskExecutorAddress = "/taskExecutor1";
- ResourceID taskExecutorResourceID = mockTaskExecutor(taskExecutorAddress);
- TestingLeaderElectionService leaderElectionService = new TestingLeaderElectionService();
- final ResourceManager resourceManager = createAndStartResourceManager(leaderElectionService);
- final UUID leaderSessionId = grantLeadership(leaderElectionService);
-
// test throw exception when receive a registration from taskExecutor which takes invalid address
String invalidAddress = "/taskExecutor2";
- Future<RegistrationResponse> invalidAddressFuture = resourceManager.registerTaskExecutor(leaderSessionId, invalidAddress, taskExecutorResourceID);
+ Future<RegistrationResponse> invalidAddressFuture =
+ resourceManager.registerTaskExecutor(leaderSessionId, invalidAddress, taskExecutorResourceID, slotReport);
assertTrue(invalidAddressFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
}
@@ -118,10 +118,11 @@ public class ResourceManagerTaskExecutorTest {
return taskExecutorResourceID;
}
- private ResourceManager createAndStartResourceManager(TestingLeaderElectionService rmLeaderElectionService) {
+ private StandaloneResourceManager createAndStartResourceManager(TestingLeaderElectionService rmLeaderElectionService) {
TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
- ResourceManager resourceManager = new StandaloneResourceManager(rpcService, highAvailabilityServices, new SimpleSlotManager());
+ StandaloneResourceManager resourceManager =
+ new TestingResourceManager(rpcService, highAvailabilityServices);
resourceManager.start();
return resourceManager;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java
new file mode 100644
index 0000000..6b4ca14
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.runtime.resourcemanager;
+
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+import org.apache.flink.runtime.rpc.RpcService;
+
+public class TestingResourceManager extends StandaloneResourceManager {
+
+ public TestingResourceManager(RpcService rpcService) {
+ this(rpcService, new TestingHighAvailabilityServices());
+ }
+
+ public TestingResourceManager(
+ RpcService rpcService,
+ HighAvailabilityServices highAvailabilityServices) {
+ this(rpcService, highAvailabilityServices, new TestingSlotManagerFactory());
+ }
+
+ public TestingResourceManager(
+ RpcService rpcService,
+ HighAvailabilityServices highAvailabilityServices,
+ SlotManagerFactory slotManagerFactory) {
+ super(rpcService, highAvailabilityServices, slotManagerFactory);
+ }
+
+ private static class TestingSlotManagerFactory implements SlotManagerFactory {
+
+ @Override
+ public SlotManager create(ResourceManagerServices rmServices) {
+ return new TestingSlotManager(rmServices);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManager.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManager.java
new file mode 100644
index 0000000..0b2c42b
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManager.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.runtime.resourcemanager;
+
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import org.apache.flink.runtime.clusterframework.types.ResourceSlot;
+import org.apache.flink.runtime.clusterframework.types.SlotID;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
+import org.mockito.Mockito;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.Executor;
+
+public class TestingSlotManager extends SlotManager {
+
+ public TestingSlotManager() {
+ this(new TestingResourceManagerServices());
+ }
+
+ public TestingSlotManager(ResourceManagerServices rmServices) {
+ super(rmServices);
+ }
+
+ @Override
+ protected ResourceSlot chooseSlotToUse(SlotRequest request, Map<SlotID, ResourceSlot> freeSlots) {
+ final Iterator<ResourceSlot> slotIterator = freeSlots.values().iterator();
+ if (slotIterator.hasNext()) {
+ return slotIterator.next();
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ protected SlotRequest chooseRequestToFulfill(ResourceSlot offeredSlot, Map<AllocationID, SlotRequest> pendingRequests) {
+ final Iterator<SlotRequest> requestIterator = pendingRequests.values().iterator();
+ if (requestIterator.hasNext()) {
+ return requestIterator.next();
+ } else {
+ return null;
+ }
+ }
+
+ private static class TestingResourceManagerServices implements ResourceManagerServices {
+
+ @Override
+ public void allocateResource(ResourceProfile resourceProfile) {
+
+ }
+
+ @Override
+ public Executor getAsyncExecutor() {
+ return Mockito.mock(Executor.class);
+ }
+
+ @Override
+ public Executor getMainThreadExecutor() {
+ return Mockito.mock(Executor.class);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
index 0fed79e..0d2b40d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
@@ -28,13 +28,16 @@ import org.apache.flink.runtime.clusterframework.types.SlotID;
import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
import org.apache.flink.runtime.resourcemanager.ResourceManagerServices;
import org.apache.flink.runtime.resourcemanager.SlotRequest;
-import org.apache.flink.runtime.resourcemanager.SlotRequestReply;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
+import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorRegistration;
+import org.apache.flink.runtime.taskexecutor.SlotReport;
import org.apache.flink.runtime.taskexecutor.SlotStatus;
import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
+import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -45,7 +48,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
public class SlotManagerTest {
@@ -59,13 +61,15 @@ public class SlotManagerTest {
private static final ResourceProfile DEFAULT_TESTING_BIG_PROFILE =
new ResourceProfile(DEFAULT_TESTING_CPU_CORES * 2, DEFAULT_TESTING_MEMORY * 2);
- private static TaskExecutorGateway taskExecutorGateway;
+ private static TaskExecutorRegistration taskExecutorRegistration;
@BeforeClass
public static void setUp() {
- taskExecutorGateway = Mockito.mock(TaskExecutorGateway.class);
- Mockito.when(taskExecutorGateway.requestSlot(any(AllocationID.class), any(UUID.class), any(Time.class)))
- .thenReturn(new FlinkCompletableFuture<SlotRequestReply>());
+ taskExecutorRegistration = Mockito.mock(TaskExecutorRegistration.class);
+ TaskExecutorGateway gateway = Mockito.mock(TaskExecutorGateway.class);
+ Mockito.when(taskExecutorRegistration.getTaskExecutorGateway()).thenReturn(gateway);
+ Mockito.when(gateway.requestSlot(any(SlotID.class), any(AllocationID.class), any(UUID.class), any(Time.class)))
+ .thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
}
/**
@@ -180,9 +184,9 @@ public class SlotManagerTest {
assertEquals(1, slotManager.getPendingRequestCount());
SlotID slotId = SlotID.generate();
- slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorGateway);
SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE);
- slotManager.updateSlotStatus(slotStatus);
+ SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
+ slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorRegistration, slotReport);
assertEquals(1, slotManager.getAllocatedSlotCount());
assertEquals(0, slotManager.getFreeSlotCount());
@@ -198,9 +202,9 @@ public class SlotManagerTest {
TestingSlotManager slotManager = new TestingSlotManager();
SlotID slotId = SlotID.generate();
- slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorGateway);
SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE);
- slotManager.updateSlotStatus(slotStatus);
+ SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
+ slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorRegistration, slotReport);
assertEquals(0, slotManager.getAllocatedSlotCount());
assertEquals(1, slotManager.getFreeSlotCount());
@@ -216,9 +220,9 @@ public class SlotManagerTest {
assertEquals(1, slotManager.getPendingRequestCount());
SlotID slotId = SlotID.generate();
- slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorGateway);
SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE);
- slotManager.updateSlotStatus(slotStatus);
+ SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
+ slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorRegistration, slotReport);
assertEquals(0, slotManager.getAllocatedSlotCount());
assertEquals(1, slotManager.getFreeSlotCount());
@@ -234,9 +238,9 @@ public class SlotManagerTest {
TestingSlotManager slotManager = new TestingSlotManager();
SlotID slotId = SlotID.generate();
- slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorGateway);
SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE, new JobID(), new AllocationID());
- slotManager.updateSlotStatus(slotStatus);
+ SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
+ slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorRegistration, slotReport);
assertEquals(1, slotManager.getAllocatedSlotCount());
assertEquals(0, slotManager.getFreeSlotCount());
@@ -244,48 +248,44 @@ public class SlotManagerTest {
}
/**
- * Tests that we had a slot in-use, and it's confirmed by SlotReport
+ * Tests that we had a slot in-use and is freed again subsequently.
*/
@Test
public void testExistingInUseSlotUpdateStatus() {
TestingSlotManager slotManager = new TestingSlotManager();
- SlotRequest request = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
- slotManager.requestSlot(request);
- // make this slot in use
SlotID slotId = SlotID.generate();
- slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorGateway);
- SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE);
- slotManager.updateSlotStatus(slotStatus);
+ SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE, new JobID(), new AllocationID());
+ SlotReport slotReport = new SlotReport(Collections.singletonList(slotStatus));
+ slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorRegistration, slotReport);
assertEquals(1, slotManager.getAllocatedSlotCount());
assertEquals(0, slotManager.getFreeSlotCount());
assertTrue(slotManager.isAllocated(slotId));
- // slot status is confirmed
- SlotStatus slotStatus2 = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE,
- request.getJobId(), request.getAllocationId());
- slotManager.updateSlotStatus(slotStatus2);
+ // slot is freed again
+ slotManager.notifySlotAvailable(slotId.getResourceID(), slotId);
- assertEquals(1, slotManager.getAllocatedSlotCount());
- assertEquals(0, slotManager.getFreeSlotCount());
- assertTrue(slotManager.isAllocated(slotId));
+ assertEquals(0, slotManager.getAllocatedSlotCount());
+ assertEquals(1, slotManager.getFreeSlotCount());
+ assertFalse(slotManager.isAllocated(slotId));
}
/**
- * Tests that we had a slot in-use, but it's empty according to the SlotReport
+ * Tests multiple slot requests with one slots.
*/
@Test
- public void testExistingInUseSlotAdjustedToEmpty() {
+ public void testMultipleSlotRequestsWithOneSlot() {
TestingSlotManager slotManager = new TestingSlotManager();
- SlotRequest request1 = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
+ final AllocationID allocationID = new AllocationID();
+
+ SlotRequest request1 = new SlotRequest(new JobID(), allocationID, DEFAULT_TESTING_PROFILE);
slotManager.requestSlot(request1);
- // make this slot in use
- SlotID slotId = SlotID.generate();
- slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorGateway);
- SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE);
- slotManager.updateSlotStatus(slotStatus);
+ final ResourceID resourceID = ResourceID.generate();
+ final SlotStatus slotStatus = new SlotStatus(new SlotID(resourceID, 0), DEFAULT_TESTING_PROFILE);
+ final SlotReport slotReport = new SlotReport(slotStatus);
+ slotManager.registerTaskExecutor(resourceID, taskExecutorRegistration, slotReport);
// another request pending
SlotRequest request2 = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
@@ -294,66 +294,20 @@ public class SlotManagerTest {
assertEquals(1, slotManager.getAllocatedSlotCount());
assertEquals(0, slotManager.getFreeSlotCount());
assertEquals(1, slotManager.getPendingRequestCount());
- assertTrue(slotManager.isAllocated(slotId));
+ assertTrue(slotManager.isAllocated(allocationID));
assertTrue(slotManager.isAllocated(request1.getAllocationId()));
-
- // but slot is reported empty again, request2 will be fulfilled, request1 will be missing
- slotManager.updateSlotStatus(slotStatus);
+ // but slot is reported empty in a report in the meantime which shouldn't affect the state
+ slotManager.notifySlotAvailable(resourceID, slotStatus.getSlotID());
assertEquals(1, slotManager.getAllocatedSlotCount());
assertEquals(0, slotManager.getFreeSlotCount());
assertEquals(0, slotManager.getPendingRequestCount());
- assertTrue(slotManager.isAllocated(slotId));
+ assertTrue(slotManager.isAllocated(slotStatus.getSlotID()));
assertTrue(slotManager.isAllocated(request2.getAllocationId()));
- }
-
- /**
- * Tests that we had a slot in use, and it's also reported in use by TaskManager, but the allocation
- * information didn't match.
- */
- @Test
- public void testExistingInUseSlotWithDifferentAllocationInfo() {
- TestingSlotManager slotManager = new TestingSlotManager();
- SlotRequest request = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
- slotManager.requestSlot(request);
-
- // make this slot in use
- SlotID slotId = SlotID.generate();
- slotManager.registerTaskExecutor(slotId.getResourceID(), taskExecutorGateway);
- SlotStatus slotStatus = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE);
- slotManager.updateSlotStatus(slotStatus);
-
- assertEquals(1, slotManager.getAllocatedSlotCount());
- assertEquals(0, slotManager.getFreeSlotCount());
- assertEquals(0, slotManager.getPendingRequestCount());
- assertTrue(slotManager.isAllocated(slotId));
- assertTrue(slotManager.isAllocated(request.getAllocationId()));
-
- SlotStatus slotStatus2 = new SlotStatus(slotId, DEFAULT_TESTING_PROFILE, new JobID(), new AllocationID());
- // update slot status with different allocation info
- slotManager.updateSlotStatus(slotStatus2);
-
- // original request is missing and won't be allocated
- assertEquals(1, slotManager.getAllocatedSlotCount());
- assertEquals(0, slotManager.getFreeSlotCount());
- assertEquals(0, slotManager.getPendingRequestCount());
- assertTrue(slotManager.isAllocated(slotId));
- assertFalse(slotManager.isAllocated(request.getAllocationId()));
- assertTrue(slotManager.isAllocated(slotStatus2.getAllocationID()));
- }
-
- /**
- * Tests that we had a free slot, and it's confirmed by SlotReport
- */
- @Test
- public void testExistingEmptySlotUpdateStatus() {
- TestingSlotManager slotManager = new TestingSlotManager();
- ResourceSlot slot = new ResourceSlot(SlotID.generate(), DEFAULT_TESTING_PROFILE, taskExecutorGateway);
- slotManager.addFreeSlot(slot);
- SlotStatus slotStatus = new SlotStatus(slot.getSlotId(), DEFAULT_TESTING_PROFILE);
- slotManager.updateSlotStatus(slotStatus);
+ // but slot is reported empty in a report in the meantime which shouldn't affect the state
+ slotManager.notifySlotAvailable(resourceID, slotStatus.getSlotID());
assertEquals(0, slotManager.getAllocatedSlotCount());
assertEquals(1, slotManager.getFreeSlotCount());
@@ -361,34 +315,12 @@ public class SlotManagerTest {
}
/**
- * Tests that we had a free slot, and it's reported in-use by TaskManager
- */
- @Test
- public void testExistingEmptySlotAdjustedToInUse() {
- TestingSlotManager slotManager = new TestingSlotManager();
- final SlotID slotID = SlotID.generate();
- slotManager.registerTaskExecutor(slotID.getResourceID(), taskExecutorGateway);
-
- ResourceSlot slot = new ResourceSlot(slotID, DEFAULT_TESTING_PROFILE, taskExecutorGateway);
- slotManager.addFreeSlot(slot);
-
- SlotStatus slotStatus = new SlotStatus(slot.getSlotId(), DEFAULT_TESTING_PROFILE,
- new JobID(), new AllocationID());
- slotManager.updateSlotStatus(slotStatus);
-
- assertEquals(1, slotManager.getAllocatedSlotCount());
- assertEquals(0, slotManager.getFreeSlotCount());
- assertEquals(0, slotManager.getPendingRequestCount());
- assertTrue(slotManager.isAllocated(slot.getSlotId()));
- }
-
- /**
* Tests that we did some allocation but failed / rejected by TaskManager, request will retry
*/
@Test
public void testSlotAllocationFailedAtTaskManager() {
TestingSlotManager slotManager = new TestingSlotManager();
- ResourceSlot slot = new ResourceSlot(SlotID.generate(), DEFAULT_TESTING_PROFILE, taskExecutorGateway);
+ ResourceSlot slot = new ResourceSlot(SlotID.generate(), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
slotManager.addFreeSlot(slot);
SlotRequest request = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
@@ -409,24 +341,31 @@ public class SlotManagerTest {
/**
* Tests that we did some allocation but failed / rejected by TaskManager, and slot is occupied by another request
+ * This can only occur after reconnect of the TaskExecutor.
*/
@Test
public void testSlotAllocationFailedAtTaskManagerOccupiedByOther() {
TestingSlotManager slotManager = new TestingSlotManager();
final SlotID slotID = SlotID.generate();
- slotManager.registerTaskExecutor(slotID.getResourceID(), taskExecutorGateway);
-
- ResourceSlot slot = new ResourceSlot(slotID, DEFAULT_TESTING_PROFILE, taskExecutorGateway);
- slotManager.addFreeSlot(slot);
+ SlotStatus slot = new SlotStatus(slotID, DEFAULT_TESTING_PROFILE);
+ SlotReport slotReport = new SlotReport(slot);
+ slotManager.registerTaskExecutor(slotID.getResourceID(), taskExecutorRegistration, slotReport);
SlotRequest request = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
slotManager.requestSlot(request);
- // slot is set empty by heartbeat
- SlotStatus slotStatus = new SlotStatus(slot.getSlotId(), slot.getResourceProfile());
- slotManager.updateSlotStatus(slotStatus);
+ assertEquals(1, slotManager.getAllocatedSlotCount());
+ assertEquals(0, slotManager.getFreeSlotCount());
+ assertEquals(0, slotManager.getPendingRequestCount());
+
+ // slot is set empty by a reconnect of the TaskExecutor
+ slotManager.registerTaskExecutor(slotID.getResourceID(), taskExecutorRegistration, slotReport);
+
+ assertEquals(0, slotManager.getAllocatedSlotCount());
+ assertEquals(1, slotManager.getFreeSlotCount());
+ assertEquals(0, slotManager.getPendingRequestCount());
- // another request took this slot
+ // another request takes the slot
SlotRequest request2 = new SlotRequest(new JobID(), new AllocationID(), DEFAULT_TESTING_PROFILE);
slotManager.requestSlot(request2);
@@ -436,12 +375,12 @@ public class SlotManagerTest {
assertFalse(slotManager.isAllocated(request.getAllocationId()));
assertTrue(slotManager.isAllocated(request2.getAllocationId()));
- // original request should be pended
- slotManager.handleSlotRequestFailedAtTaskManager(request, slot.getSlotId());
+ // original request should be retried
+ slotManager.handleSlotRequestFailedAtTaskManager(request, slotID);
assertEquals(1, slotManager.getAllocatedSlotCount());
assertEquals(0, slotManager.getFreeSlotCount());
- assertEquals(1, slotManager.getPendingRequestCount());
+ assertEquals(0, slotManager.getPendingRequestCount());
assertFalse(slotManager.isAllocated(request.getAllocationId()));
assertTrue(slotManager.isAllocated(request2.getAllocationId()));
}
@@ -453,10 +392,10 @@ public class SlotManagerTest {
ResourceID resource1 = ResourceID.generate();
ResourceID resource2 = ResourceID.generate();
- ResourceSlot slot11 = new ResourceSlot(new SlotID(resource1, 1), DEFAULT_TESTING_PROFILE, taskExecutorGateway);
- ResourceSlot slot12 = new ResourceSlot(new SlotID(resource1, 2), DEFAULT_TESTING_PROFILE, taskExecutorGateway);
- ResourceSlot slot21 = new ResourceSlot(new SlotID(resource2, 1), DEFAULT_TESTING_PROFILE, taskExecutorGateway);
- ResourceSlot slot22 = new ResourceSlot(new SlotID(resource2, 2), DEFAULT_TESTING_PROFILE, taskExecutorGateway);
+ ResourceSlot slot11 = new ResourceSlot(new SlotID(resource1, 1), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
+ ResourceSlot slot12 = new ResourceSlot(new SlotID(resource1, 2), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
+ ResourceSlot slot21 = new ResourceSlot(new SlotID(resource2, 1), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
+ ResourceSlot slot22 = new ResourceSlot(new SlotID(resource2, 2), DEFAULT_TESTING_PROFILE, taskExecutorRegistration);
slotManager.addFreeSlot(slot11);
slotManager.addFreeSlot(slot21);
@@ -499,7 +438,7 @@ public class SlotManagerTest {
final int freeSlotNum)
{
for (int i = 0; i < freeSlotNum; ++i) {
- slotManager.addFreeSlot(new ResourceSlot(SlotID.generate(), new ResourceProfile(resourceProfile), taskExecutorGateway));
+ slotManager.addFreeSlot(new ResourceSlot(SlotID.generate(), new ResourceProfile(resourceProfile), taskExecutorRegistration));
}
}
@@ -507,13 +446,13 @@ public class SlotManagerTest {
// testing classes
// ------------------------------------------------------------------------
- private static class TestingSlotManager extends SlotManager implements ResourceManagerServices {
+ private static class TestingSlotManager extends SlotManager {
- private final List<ResourceProfile> allocatedContainers;
+ private static TestingRmServices testingRmServices = new TestingRmServices();
TestingSlotManager() {
- this.allocatedContainers = new LinkedList<>();
- setupResourceManagerServices(this);
+ super(testingRmServices);
+ testingRmServices.allocatedContainers.clear();
}
/**
@@ -552,24 +491,34 @@ public class SlotManagerTest {
return null;
}
- @Override
- public void allocateResource(ResourceProfile resourceProfile) {
- allocatedContainers.add(resourceProfile);
+ List<ResourceProfile> getAllocatedContainers() {
+ return testingRmServices.allocatedContainers;
}
- @Override
- public Executor getAsyncExecutor() {
- return Mockito.mock(Executor.class);
- }
- @Override
- public Executor getExecutor() {
- return Mockito.mock(Executor.class);
- }
+ private static class TestingRmServices implements ResourceManagerServices {
- List<ResourceProfile> getAllocatedContainers() {
- return allocatedContainers;
- }
+ private List<ResourceProfile> allocatedContainers;
+
+ public TestingRmServices() {
+ this.allocatedContainers = new LinkedList<>();
+ }
+
+ @Override
+ public void allocateResource(ResourceProfile resourceProfile) {
+ allocatedContainers.add(resourceProfile);
+ }
+ @Override
+ public Executor getAsyncExecutor() {
+ return Mockito.mock(Executor.class);
+ }
+
+ @Override
+ public Executor getMainThreadExecutor() {
+ return Mockito.mock(Executor.class);
+ }
+
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
index a87fe42..24d959e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
@@ -25,12 +25,20 @@ import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.clusterframework.types.SlotID;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
import org.apache.flink.runtime.jobmaster.JobMasterGateway;
import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
import org.apache.flink.runtime.registration.RegistrationResponse;
-import org.apache.flink.runtime.resourcemanager.*;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerServices;
+import org.apache.flink.runtime.resourcemanager.SlotRequest;
+import org.apache.flink.runtime.resourcemanager.TestingResourceManager;
+import org.apache.flink.runtime.resourcemanager.TestingSlotManager;
+import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestReply;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
+import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorRegistration;
+import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.rpc.TestingSerialRpcService;
import org.apache.flink.runtime.taskexecutor.SlotReport;
import org.apache.flink.runtime.taskexecutor.SlotStatus;
@@ -94,9 +102,9 @@ public class SlotProtocolTest extends TestLogger {
TestingLeaderElectionService rmLeaderElectionService =
configureHA(testingHaServices, jobID, rmAddress, rmLeaderID, jmAddress, jmLeaderID);
- SlotManager slotManager = Mockito.spy(new SimpleSlotManager());
- ResourceManager resourceManager =
- Mockito.spy(new StandaloneResourceManager(testRpcService, testingHaServices, slotManager));
+ final TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
+ SpiedResourceManager resourceManager =
+ new SpiedResourceManager(testRpcService, testingHaServices, slotManagerFactory);
resourceManager.start();
rmLeaderElectionService.isLeader(rmLeaderID);
@@ -108,11 +116,13 @@ public class SlotProtocolTest extends TestLogger {
Assert.fail("JobManager registration Future didn't become ready.");
}
+ final SlotManager slotManager = slotManagerFactory.slotManager;
+
final AllocationID allocationID = new AllocationID();
final ResourceProfile resourceProfile = new ResourceProfile(1.0, 100);
SlotRequest slotRequest = new SlotRequest(jobID, allocationID, resourceProfile);
- SlotRequestReply slotRequestReply =
+ RMSlotRequestReply slotRequestReply =
resourceManager.requestSlot(jmLeaderID, rmLeaderID, slotRequest);
// 1) SlotRequest is routed to the SlotManager
@@ -124,15 +134,18 @@ public class SlotProtocolTest extends TestLogger {
allocationID);
// 3) SlotRequest leads to a container allocation
- verify(resourceManager, timeout(5000)).startNewWorker(resourceProfile);
+ Assert.assertEquals(1, resourceManager.startNewWorkerCalled);
Assert.assertFalse(slotManager.isAllocated(allocationID));
// slot becomes available
final String tmAddress = "/tm1";
TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
- Mockito.when(taskExecutorGateway.requestSlot(any(AllocationID.class), any(UUID.class), any(Time.class)))
- .thenReturn(new FlinkCompletableFuture<SlotRequestReply>());
+ Mockito
+ .when(
+ taskExecutorGateway
+ .requestSlot(any(SlotID.class), any(AllocationID.class), any(UUID.class), any(Time.class)))
+ .thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
testRpcService.registerGateway(tmAddress, taskExecutorGateway);
final ResourceID resourceID = ResourceID.generate();
@@ -141,13 +154,14 @@ public class SlotProtocolTest extends TestLogger {
final SlotStatus slotStatus =
new SlotStatus(slotID, resourceProfile);
final SlotReport slotReport =
- new SlotReport(Collections.singletonList(slotStatus), resourceID);
+ new SlotReport(Collections.singletonList(slotStatus));
// register slot at SlotManager
- slotManager.registerTaskExecutor(resourceID, taskExecutorGateway);
- slotManager.updateSlotStatus(slotReport);
+ slotManager.registerTaskExecutor(
+ resourceID, new TaskExecutorRegistration(taskExecutorGateway), slotReport);
// 4) Slot becomes available and TaskExecutor gets a SlotRequest
- verify(taskExecutorGateway, timeout(5000)).requestSlot(eq(allocationID), any(UUID.class), any(Time.class));
+ verify(taskExecutorGateway, timeout(5000))
+ .requestSlot(eq(slotID), eq(allocationID), any(UUID.class), any(Time.class));
}
/**
@@ -173,13 +187,15 @@ public class SlotProtocolTest extends TestLogger {
configureHA(testingHaServices, jobID, rmAddress, rmLeaderID, jmAddress, jmLeaderID);
TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
- Mockito.when(taskExecutorGateway.requestSlot(any(AllocationID.class), any(UUID.class), any(Time.class)))
- .thenReturn(new FlinkCompletableFuture<SlotRequestReply>());
+ Mockito.when(
+ taskExecutorGateway
+ .requestSlot(any(SlotID.class), any(AllocationID.class), any(UUID.class), any(Time.class)))
+ .thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
testRpcService.registerGateway(tmAddress, taskExecutorGateway);
- SlotManager slotManager = Mockito.spy(new SimpleSlotManager());
- ResourceManager resourceManager =
- Mockito.spy(new StandaloneResourceManager(testRpcService, testingHaServices, slotManager));
+ TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
+ TestingResourceManager resourceManager =
+ Mockito.spy(new TestingResourceManager(testRpcService, testingHaServices, slotManagerFactory));
resourceManager.start();
rmLeaderElectionService.isLeader(rmLeaderID);
@@ -191,6 +207,8 @@ public class SlotProtocolTest extends TestLogger {
Assert.fail("JobManager registration Future didn't become ready.");
}
+ final SlotManager slotManager = slotManagerFactory.slotManager;
+
final ResourceID resourceID = ResourceID.generate();
final AllocationID allocationID = new AllocationID();
final ResourceProfile resourceProfile = new ResourceProfile(1.0, 100);
@@ -199,13 +217,13 @@ public class SlotProtocolTest extends TestLogger {
final SlotStatus slotStatus =
new SlotStatus(slotID, resourceProfile);
final SlotReport slotReport =
- new SlotReport(Collections.singletonList(slotStatus), resourceID);
+ new SlotReport(Collections.singletonList(slotStatus));
// register slot at SlotManager
- slotManager.registerTaskExecutor(resourceID, taskExecutorGateway);
- slotManager.updateSlotStatus(slotReport);
+ slotManager.registerTaskExecutor(
+ resourceID, new TaskExecutorRegistration(taskExecutorGateway), slotReport);
SlotRequest slotRequest = new SlotRequest(jobID, allocationID, resourceProfile);
- SlotRequestReply slotRequestReply =
+ RMSlotRequestReply slotRequestReply =
resourceManager.requestSlot(jmLeaderID, rmLeaderID, slotRequest);
// 1) a SlotRequest is routed to the SlotManager
@@ -220,9 +238,9 @@ public class SlotProtocolTest extends TestLogger {
Assert.assertTrue(slotManager.isAllocated(slotID));
Assert.assertTrue(slotManager.isAllocated(allocationID));
-
// 4) a SlotRequest is routed to the TaskExecutor
- verify(taskExecutorGateway, timeout(5000)).requestSlot(eq(allocationID), any(UUID.class), any(Time.class));
+ verify(taskExecutorGateway, timeout(5000))
+ .requestSlot(eq(slotID), eq(allocationID), any(UUID.class), any(Time.class));
}
private static TestingLeaderElectionService configureHA(
@@ -240,4 +258,32 @@ public class SlotProtocolTest extends TestLogger {
return rmLeaderElectionService;
}
+ private static class SpiedResourceManager extends TestingResourceManager {
+
+ private int startNewWorkerCalled = 0;
+
+ public SpiedResourceManager(
+ RpcService rpcService,
+ HighAvailabilityServices highAvailabilityServices,
+ SlotManagerFactory slotManagerFactory) {
+ super(rpcService, highAvailabilityServices, slotManagerFactory);
+ }
+
+
+ @Override
+ public void startNewWorker(ResourceProfile resourceProfile) {
+ startNewWorkerCalled++;
+ }
+ }
+
+ private static class TestingSlotManagerFactory implements SlotManagerFactory {
+
+ private SlotManager slotManager;
+
+ @Override
+ public SlotManager create(ResourceManagerServices rmServices) {
+ this.slotManager = Mockito.spy(new TestingSlotManager(rmServices));
+ return this.slotManager;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8adceede/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index 9c1f288..7710fa9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -19,7 +19,9 @@
package org.apache.flink.runtime.taskexecutor;
import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.clusterframework.types.SlotID;
import org.apache.flink.runtime.highavailability.NonHaServices;
import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
@@ -28,11 +30,15 @@ import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
import org.apache.flink.runtime.memory.MemoryManager;
import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRegistered;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestRejected;
+import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.TestingSerialRpcService;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.util.TestLogger;
+import org.hamcrest.Matchers;
import org.junit.Test;
import org.powermock.api.mockito.PowerMockito;
@@ -76,7 +82,7 @@ public class TaskExecutorTest extends TestLogger {
String taskManagerAddress = taskManager.getAddress();
verify(rmGateway).registerTaskExecutor(
- any(UUID.class), eq(taskManagerAddress), eq(resourceID), any(Time.class));
+ any(UUID.class), eq(taskManagerAddress), eq(resourceID), any(SlotReport.class), any(Time.class));
}
finally {
rpc.stopService();
@@ -132,7 +138,7 @@ public class TaskExecutorTest extends TestLogger {
testLeaderService.notifyListener(address1, leaderId1);
verify(rmGateway1).registerTaskExecutor(
- eq(leaderId1), eq(taskManagerAddress), eq(resourceID), any(Time.class));
+ eq(leaderId1), eq(taskManagerAddress), eq(resourceID), any(SlotReport.class), any(Time.class));
assertNotNull(taskManager.getResourceManagerConnection());
// cancel the leader
@@ -142,11 +148,95 @@ public class TaskExecutorTest extends TestLogger {
testLeaderService.notifyListener(address2, leaderId2);
verify(rmGateway2).registerTaskExecutor(
- eq(leaderId2), eq(taskManagerAddress), eq(resourceID), any(Time.class));
+ eq(leaderId2), eq(taskManagerAddress), eq(resourceID), any(SlotReport.class), any(Time.class));
assertNotNull(taskManager.getResourceManagerConnection());
}
finally {
rpc.stopService();
}
}
+
+ /**
+ * Tests that all allocation requests for slots are ignored if the slot has been reported as
+ * free by the TaskExecutor but this report hasn't been confirmed by the ResourceManager.
+ *
+ * This is essential for the correctness of the state of the ResourceManager.
+ */
+ @Test
+ public void testRejectAllocationRequestsForOutOfSyncSlots() {
+ final ResourceID resourceID = ResourceID.generate();
+
+ final String address1 = "/resource/manager/address/one";
+ final UUID leaderId = UUID.randomUUID();
+
+ final TestingSerialRpcService rpc = new TestingSerialRpcService();
+ try {
+ // register the mock resource manager gateways
+ ResourceManagerGateway rmGateway1 = mock(ResourceManagerGateway.class);
+ rpc.registerGateway(address1, rmGateway1);
+
+ TestingLeaderRetrievalService testLeaderService = new TestingLeaderRetrievalService();
+
+ TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServices();
+ haServices.setResourceManagerLeaderRetriever(testLeaderService);
+
+ TaskManagerConfiguration taskManagerServicesConfiguration = mock(TaskManagerConfiguration.class);
+ PowerMockito.when(taskManagerServicesConfiguration.getNumberSlots()).thenReturn(1);
+
+ TaskManagerLocation taskManagerLocation = mock(TaskManagerLocation.class);
+ when(taskManagerLocation.getResourceID()).thenReturn(resourceID);
+
+ TaskExecutor taskManager = new TaskExecutor(
+ taskManagerServicesConfiguration,
+ taskManagerLocation,
+ rpc,
+ mock(MemoryManager.class),
+ mock(IOManager.class),
+ mock(NetworkEnvironment.class),
+ haServices,
+ mock(MetricRegistry.class),
+ mock(FatalErrorHandler.class));
+
+ taskManager.start();
+ String taskManagerAddress = taskManager.getAddress();
+
+ // no connection initially, since there is no leader
+ assertNull(taskManager.getResourceManagerConnection());
+
+ // define a leader and see that a registration happens
+ testLeaderService.notifyListener(address1, leaderId);
+
+ verify(rmGateway1).registerTaskExecutor(
+ eq(leaderId), eq(taskManagerAddress), eq(resourceID), any(SlotReport.class), any(Time.class));
+ assertNotNull(taskManager.getResourceManagerConnection());
+
+ // test that allocating a slot works
+ final SlotID slotID = new SlotID(resourceID, 0);
+ TMSlotRequestReply tmSlotRequestReply = taskManager.requestSlot(slotID, new AllocationID(), leaderId);
+ assertTrue(tmSlotRequestReply instanceof TMSlotRequestRegistered);
+
+ // test that we can't allocate slots which are blacklisted due to pending confirmation of the RM
+ final SlotID unconfirmedFreeSlotID = new SlotID(resourceID, 1);
+ taskManager.addUnconfirmedFreeSlotNotification(unconfirmedFreeSlotID);
+ TMSlotRequestReply tmSlotRequestReply2 =
+ taskManager.requestSlot(unconfirmedFreeSlotID, new AllocationID(), leaderId);
+ assertTrue(tmSlotRequestReply2 instanceof TMSlotRequestRejected);
+
+ // re-register
+ verify(rmGateway1).registerTaskExecutor(
+ eq(leaderId), eq(taskManagerAddress), eq(resourceID), any(SlotReport.class), any(Time.class));
+ testLeaderService.notifyListener(address1, leaderId);
+
+ // now we should be successful because the slots status has been synced
+ // test that we can't allocate slots which are blacklisted due to pending confirmation of the RM
+ TMSlotRequestReply tmSlotRequestReply3 =
+ taskManager.requestSlot(unconfirmedFreeSlotID, new AllocationID(), leaderId);
+ assertTrue(tmSlotRequestReply3 instanceof TMSlotRequestRegistered);
+
+ }
+ finally {
+ rpc.stopService();
+ }
+
+ }
}
[33/50] [abbrv] flink git commit: [FLINK-4351] [cluster management]
JobManager handle TaskManager's registration
Posted by tr...@apache.org.
[FLINK-4351] [cluster management] JobManager handle TaskManager's registration
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/60ee9431
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/60ee9431
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/60ee9431
Branch: refs/heads/flip-6
Commit: 60ee9431291cc6b73f2f51658842fc407f2d2ee8
Parents: 0a02232
Author: Kurt Young <yk...@gmail.com>
Authored: Sun Oct 16 23:00:57 2016 +0800
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:32 2016 +0100
----------------------------------------------------------------------
.../flink/runtime/jobmaster/JobMaster.java | 75 +++++++++++++++++---
.../runtime/jobmaster/JobMasterGateway.java | 15 ++--
.../runtime/taskexecutor/JobLeaderService.java | 55 ++++++--------
.../runtime/taskexecutor/TaskExecutor.java | 2 +-
.../taskexecutor/TaskManagerServices.java | 2 +-
.../runtime/taskexecutor/TaskExecutorTest.java | 10 ++-
6 files changed, 102 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/60ee9431/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 05c20d3..8cb9946 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.jobmaster;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.time.Time;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.io.InputSplit;
import org.apache.flink.core.io.InputSplitAssigner;
@@ -36,7 +37,9 @@ import org.apache.flink.runtime.client.SerializedJobExecutionResult;
import org.apache.flink.runtime.clusterframework.types.AllocationID;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.concurrent.ApplyFunction;
+import org.apache.flink.runtime.concurrent.BiFunction;
import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager;
import org.apache.flink.runtime.executiongraph.Execution;
@@ -81,7 +84,9 @@ import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.rpc.StartStoppable;
import org.apache.flink.runtime.state.CheckpointStateHandles;
import org.apache.flink.runtime.state.KeyGroupRange;
+import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
+import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.runtime.util.SerializedThrowable;
import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.SerializedValue;
@@ -89,8 +94,10 @@ import org.slf4j.Logger;
import javax.annotation.Nullable;
import java.io.IOException;
+import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.Callable;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
@@ -122,6 +129,8 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
/** Configuration of the JobManager */
private final Configuration configuration;
+ private final Time rpcTimeout;
+
/** Service to contend for and retrieve the leadership of JM and RM */
private final HighAvailabilityServices highAvailabilityServices;
@@ -152,7 +161,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
private volatile UUID leaderSessionID;
- // --------- resource manager --------
+ // --------- ResourceManager --------
/** Leader retriever service used to locate ResourceManager's address */
private LeaderRetrievalService resourceManagerLeaderRetriever;
@@ -160,6 +169,9 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
/** Connection with ResourceManager, null if not located address yet or we close it initiative */
private ResourceManagerConnection resourceManagerConnection;
+ // --------- TaskManagers --------
+
+ private final Map<ResourceID, Tuple2<TaskManagerLocation, TaskExecutorGateway>> registeredTaskManagers;
// ------------------------------------------------------------------------
@@ -181,6 +193,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
this.jobGraph = checkNotNull(jobGraph);
this.configuration = checkNotNull(configuration);
+ this.rpcTimeout = rpcAskTimeout;
this.highAvailabilityServices = checkNotNull(highAvailabilityService);
this.libraryCacheManager = checkNotNull(libraryCacheManager);
this.executionContext = checkNotNull(executorService);
@@ -243,6 +256,8 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
this.slotPool = new SlotPool(executorService);
this.allocationTimeout = Time.of(5, TimeUnit.SECONDS);
+
+ this.registeredTaskManagers = new HashMap<>(4);
}
//----------------------------------------------------------------------------------------------
@@ -379,8 +394,10 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
}
closeResourceManagerConnection();
- // TODO: disconnect from all registered task managers
-
+ for (ResourceID taskManagerId : registeredTaskManagers.keySet()) {
+ slotPool.releaseResource(taskManagerId);
+ }
+ registeredTaskManagers.clear();
}
//----------------------------------------------------------------------------------------------
@@ -662,11 +679,53 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
}
@RpcMethod
- public RegistrationResponse registerTaskManager(
- final String taskManagerAddress,
- final ResourceID taskManagerProcessId,
- final UUID leaderId) {
- throw new UnsupportedOperationException("Has to be implemented.");
+ public Future<RegistrationResponse> registerTaskManager(
+ final TaskManagerLocation taskManagerLocation,
+ final UUID leaderId) throws Exception
+ {
+ if (!JobMaster.this.leaderSessionID.equals(leaderId)) {
+ log.warn("Discard registration from TaskExecutor {} at ({}) because the expected " +
+ "leader session ID {} did not equal the received leader session ID {}.",
+ taskManagerLocation.getResourceID(), taskManagerLocation.addressString(),
+ JobMaster.this.leaderSessionID, leaderId);
+ throw new Exception("Leader id not match, expected: " + JobMaster.this.leaderSessionID
+ + ", actual: " + leaderId);
+ }
+
+ final ResourceID taskManagerId = taskManagerLocation.getResourceID();
+
+ if (registeredTaskManagers.containsKey(taskManagerId)) {
+ final RegistrationResponse response = new JMTMRegistrationSuccess(
+ taskManagerId, libraryCacheManager.getBlobServerPort());
+ return FlinkCompletableFuture.completed(response);
+ } else {
+ return getRpcService().execute(new Callable<TaskExecutorGateway>() {
+ @Override
+ public TaskExecutorGateway call() throws Exception {
+ return getRpcService().connect(taskManagerLocation.addressString(), TaskExecutorGateway.class)
+ .get(rpcTimeout.getSize(), rpcTimeout.getUnit());
+ }
+ }).handleAsync(new BiFunction<TaskExecutorGateway, Throwable, RegistrationResponse>() {
+ @Override
+ public RegistrationResponse apply(TaskExecutorGateway taskExecutorGateway, Throwable throwable) {
+ if (throwable != null) {
+ return new RegistrationResponse.Decline(throwable.getMessage());
+ }
+
+ if (!JobMaster.this.leaderSessionID.equals(leaderId)) {
+ log.warn("Discard registration from TaskExecutor {} at ({}) because the expected " +
+ "leader session ID {} did not equal the received leader session ID {}.",
+ taskManagerLocation.getResourceID(), taskManagerLocation.addressString(),
+ JobMaster.this.leaderSessionID, leaderId);
+ return new RegistrationResponse.Decline("Invalid leader session id");
+ }
+
+ slotPool.registerResource(taskManagerId);
+ registeredTaskManagers.put(taskManagerId, Tuple2.of(taskManagerLocation, taskExecutorGateway));
+ return new JMTMRegistrationSuccess(taskManagerId, libraryCacheManager.getBlobServerPort());
+ }
+ }, getMainThreadExecutor());
+ }
}
//----------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/60ee9431/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
index 0f155a4..4c85839 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
@@ -41,6 +41,7 @@ import org.apache.flink.runtime.rpc.RpcTimeout;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.KvState;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
+import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import java.util.UUID;
@@ -195,15 +196,13 @@ public interface JobMasterGateway extends CheckpointCoordinatorGateway {
/**
* Register the task manager at the job manager.
*
- * @param taskManagerAddress address of the task manager
- * @param taskManagerProcessId identifying the task manager
- * @param leaderId identifying the job leader
- * @param timeout for the rpc call
+ * @param taskManagerLocation location of the task manager
+ * @param leaderId identifying the job leader
+ * @param timeout for the rpc call
* @return Future registration response indicating whether the registration was successful or not
*/
Future<RegistrationResponse> registerTaskManager(
- final String taskManagerAddress,
- final ResourceID taskManagerProcessId,
- final UUID leaderId,
- @RpcTimeout final Time timeout);
+ final TaskManagerLocation taskManagerLocation,
+ final UUID leaderId,
+ @RpcTimeout final Time timeout);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/60ee9431/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
index e7f52e2..14d36ab 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
@@ -21,7 +21,6 @@ package org.apache.flink.runtime.taskexecutor;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.jobmaster.JMTMRegistrationSuccess;
@@ -32,6 +31,7 @@ import org.apache.flink.runtime.registration.RegisteredRpcConnection;
import org.apache.flink.runtime.registration.RegistrationResponse;
import org.apache.flink.runtime.registration.RetryingRegistration;
import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,8 +53,8 @@ public class JobLeaderService {
private static final Logger LOG = LoggerFactory.getLogger(JobLeaderService.class);
- /** Process id of the owning process */
- private final ResourceID ownerProcessId;
+ /** Self's location, used for the job manager connection */
+ private final TaskManagerLocation ownLocation;
/** The leader retrieval service and listener for each registered job */
private final Map<JobID, Tuple2<LeaderRetrievalService, JobLeaderService.JobManagerLeaderListener>> jobLeaderServices;
@@ -62,9 +62,6 @@ public class JobLeaderService {
/** Internal state of the service */
private volatile JobLeaderService.State state;
- /** Address of the owner of this service. This address is used for the job manager connection */
- private String ownerAddress;
-
/** Rpc service to use for establishing connections */
private RpcService rpcService;
@@ -74,14 +71,13 @@ public class JobLeaderService {
/** Job leader listener listening for job leader changes */
private JobLeaderListener jobLeaderListener;
- public JobLeaderService(ResourceID ownerProcessId) {
- this.ownerProcessId = Preconditions.checkNotNull(ownerProcessId);
+ public JobLeaderService(TaskManagerLocation location) {
+ this.ownLocation = Preconditions.checkNotNull(location);
jobLeaderServices = new HashMap<>(4);
state = JobLeaderService.State.CREATED;
- ownerAddress = null;
rpcService = null;
highAvailabilityServices = null;
jobLeaderListener = null;
@@ -94,13 +90,11 @@ public class JobLeaderService {
/**
* Start the job leader service with the given services.
*
- * @param initialOwnerAddress to be used for establishing connections (source address)
* @param initialRpcService to be used to create rpc connections
* @param initialHighAvailabilityServices to create leader retrieval services for the different jobs
* @param initialJobLeaderListener listening for job leader changes
*/
public void start(
- final String initialOwnerAddress,
final RpcService initialRpcService,
final HighAvailabilityServices initialHighAvailabilityServices,
final JobLeaderListener initialJobLeaderListener) {
@@ -110,7 +104,6 @@ public class JobLeaderService {
} else {
LOG.info("Start job leader service.");
- this.ownerAddress = Preconditions.checkNotNull(initialOwnerAddress);
this.rpcService = Preconditions.checkNotNull(initialRpcService);
this.highAvailabilityServices = Preconditions.checkNotNull(initialHighAvailabilityServices);
this.jobLeaderListener = Preconditions.checkNotNull(initialJobLeaderListener);
@@ -311,14 +304,13 @@ public class JobLeaderService {
@Override
protected RetryingRegistration<JobMasterGateway, JMTMRegistrationSuccess> generateRegistration() {
return new JobLeaderService.JobManagerRetryingRegistration(
- LOG,
- rpcService,
- "JobManager",
- JobMasterGateway.class,
- getTargetAddress(),
- getTargetLeaderId(),
- ownerAddress,
- ownerProcessId);
+ LOG,
+ rpcService,
+ "JobManager",
+ JobMasterGateway.class,
+ getTargetAddress(),
+ getTargetLeaderId(),
+ ownLocation);
}
@Override
@@ -349,10 +341,11 @@ public class JobLeaderService {
/**
* Retrying registration for the job manager <--> task manager connection.
*/
- private static final class JobManagerRetryingRegistration extends RetryingRegistration<JobMasterGateway, JMTMRegistrationSuccess> {
+ private static final class JobManagerRetryingRegistration
+ extends RetryingRegistration<JobMasterGateway, JMTMRegistrationSuccess>
+ {
- private final String taskManagerAddress;
- private final ResourceID taskManagerProcessId;
+ private final TaskManagerLocation taskManagerLocation;
JobManagerRetryingRegistration(
Logger log,
@@ -361,22 +354,18 @@ public class JobLeaderService {
Class<JobMasterGateway> targetType,
String targetAddress,
UUID leaderId,
- String taskManagerAddress,
- ResourceID taskManagerProcessId) {
+ TaskManagerLocation taskManagerLocation) {
super(log, rpcService, targetName, targetType, targetAddress, leaderId);
- this.taskManagerAddress = Preconditions.checkNotNull(taskManagerAddress);
- this.taskManagerProcessId = Preconditions.checkNotNull(taskManagerProcessId);
+ this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
}
@Override
- protected Future<RegistrationResponse> invokeRegistration(JobMasterGateway gateway, UUID leaderId, long timeoutMillis) throws Exception {
- return gateway.registerTaskManager(
- taskManagerAddress,
- taskManagerProcessId,
- leaderId,
- Time.milliseconds(timeoutMillis));
+ protected Future<RegistrationResponse> invokeRegistration(
+ JobMasterGateway gateway, UUID leaderId, long timeoutMillis) throws Exception
+ {
+ return gateway.registerTaskManager(taskManagerLocation, leaderId, Time.milliseconds(timeoutMillis));
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/60ee9431/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index 3e3a544..1201281 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -206,7 +206,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
taskSlotTable.start(new SlotActionsImpl());
// start the job leader service
- jobLeaderService.start(getAddress(), getRpcService(), haServices, new JobLeaderListenerImpl());
+ jobLeaderService.start(getRpcService(), haServices, new JobLeaderListenerImpl());
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/60ee9431/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
index 7575ba3..e8de1b1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java
@@ -207,7 +207,7 @@ public class TaskManagerServices {
final JobManagerTable jobManagerTable = new JobManagerTable();
- final JobLeaderService jobLeaderService = new JobLeaderService(resourceID);
+ final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation);
return new TaskManagerServices(
taskManagerLocation,
http://git-wip-us.apache.org/repos/asf/flink/blob/60ee9431/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index 23c6833..2220f12 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -350,7 +350,7 @@ public class TaskExecutorTest extends TestLogger {
final TimerService<AllocationID> timerService = mock(TimerService.class);
final TaskSlotTable taskSlotTable = new TaskSlotTable(Arrays.asList(mock(ResourceProfile.class)), timerService);
final JobManagerTable jobManagerTable = new JobManagerTable();
- final JobLeaderService jobLeaderService = new JobLeaderService(resourceId);
+ final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation);
final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
final TestingLeaderRetrievalService resourceManagerLeaderRetrievalService = new TestingLeaderRetrievalService();
@@ -379,8 +379,7 @@ public class TaskExecutorTest extends TestLogger {
final JobMasterGateway jobMasterGateway = mock(JobMasterGateway.class);
when(jobMasterGateway.registerTaskManager(
- any(String.class),
- eq(resourceId),
+ eq(taskManagerLocation),
eq(jobManagerLeaderId),
any(Time.class)
)).thenReturn(FlinkCompletableFuture.<RegistrationResponse>completed(new JMTMRegistrationSuccess(jmResourceId, blobPort)));
@@ -451,7 +450,7 @@ public class TaskExecutorTest extends TestLogger {
final TimerService<AllocationID> timerService = mock(TimerService.class);
final TaskSlotTable taskSlotTable = new TaskSlotTable(Arrays.asList(mock(ResourceProfile.class), mock(ResourceProfile.class)), timerService);
final JobManagerTable jobManagerTable = new JobManagerTable();
- final JobLeaderService jobLeaderService = new JobLeaderService(resourceId);
+ final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation);
final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
final String resourceManagerAddress = "rm";
@@ -484,8 +483,7 @@ public class TaskExecutorTest extends TestLogger {
final JobMasterGateway jobMasterGateway = mock(JobMasterGateway.class);
when(jobMasterGateway.registerTaskManager(
- any(String.class),
- eq(resourceId),
+ eq(taskManagerLocation),
eq(jobManagerLeaderId),
any(Time.class)
)).thenReturn(FlinkCompletableFuture.<RegistrationResponse>completed(new JMTMRegistrationSuccess(jmResourceId, blobPort)));
[38/50] [abbrv] flink git commit: [FLINK-4836] [cluster management]
Start ResourceManager and TaskManager services in MiniCluster
Posted by tr...@apache.org.
[FLINK-4836] [cluster management] Start ResourceManager and TaskManager services in MiniCluster
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2e067fc2
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2e067fc2
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2e067fc2
Branch: refs/heads/flip-6
Commit: 2e067fc2d5c898d4274f2666fd44bac064fcd693
Parents: e61ade6
Author: Till Rohrmann <tr...@apache.org>
Authored: Mon Oct 17 15:16:59 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:33 2016 +0100
----------------------------------------------------------------------
.../flink/runtime/minicluster/MiniCluster.java | 58 +++++++++++++++++++-
.../minicluster/MiniClusterConfiguration.java | 20 ++++++-
.../minicluster/MiniClusterJobDispatcher.java | 2 +-
.../runtime/taskexecutor/TaskManagerRunner.java | 3 +-
.../runtime/minicluster/MiniClusterITCase.java | 2 +-
5 files changed, 78 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2e067fc2/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
index d85234d..1ffcd12 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
@@ -26,6 +26,7 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.UnmodifiableConfiguration;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
import org.apache.flink.runtime.jobgraph.JobGraph;
@@ -33,10 +34,13 @@ import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
+import org.apache.flink.runtime.taskexecutor.TaskManagerRunner;
import org.apache.flink.util.ExceptionUtils;
import javax.annotation.concurrent.GuardedBy;
+import java.util.UUID;
+
import static org.apache.flink.util.ExceptionUtils.firstOrSuppressed;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.flink.util.Preconditions.checkState;
@@ -63,9 +67,15 @@ public class MiniCluster {
private RpcService[] taskManagerRpcServices;
@GuardedBy("lock")
+ private RpcService[] resourceManagerRpcServices;
+
+ @GuardedBy("lock")
private HighAvailabilityServices haServices;
@GuardedBy("lock")
+ private TaskManagerRunner[] taskManagerRunners;
+
+ @GuardedBy("lock")
private MiniClusterJobDispatcher jobDispatcher;
/** Flag marking the mini cluster as started/running */
@@ -143,6 +153,7 @@ public class MiniCluster {
final Time rpcTimeout = config.getRpcTimeout();
final int numJobManagers = config.getNumJobManagers();
final int numTaskManagers = config.getNumTaskManagers();
+ final int numResourceManagers = config.getNumResourceManagers();
final boolean singleRpc = config.getUseSingleRpcSystem();
try {
@@ -150,6 +161,7 @@ public class MiniCluster {
RpcService[] jobManagerRpcServices = new RpcService[numJobManagers];
RpcService[] taskManagerRpcServices = new RpcService[numTaskManagers];
+ RpcService[] resourceManagerRpcServices = new RpcService[numResourceManagers];
// bring up all the RPC services
if (singleRpc) {
@@ -163,11 +175,19 @@ public class MiniCluster {
for (int i = 0; i < numTaskManagers; i++) {
taskManagerRpcServices[i] = commonRpcService;
}
+ for (int i = 0; i < numResourceManagers; i++) {
+ resourceManagerRpcServices[i] = commonRpcService;
+ }
+
+ this.resourceManagerRpcServices = null;
+ this.jobManagerRpcServices = null;
+ this.taskManagerRpcServices = null;
}
else {
// start a new service per component, possibly with custom bind addresses
final String jobManagerBindAddress = config.getJobManagerBindAddress();
final String taskManagerBindAddress = config.getTaskManagerBindAddress();
+ final String resourceManagerBindAddress = config.getResourceManagerBindAddress();
for (int i = 0; i < numJobManagers; i++) {
jobManagerRpcServices[i] = createRpcService(
@@ -179,13 +199,23 @@ public class MiniCluster {
configuration, rpcTimeout, true, taskManagerBindAddress);
}
+ for (int i = 0; i < numResourceManagers; i++) {
+ resourceManagerRpcServices[i] = createRpcService(
+ configuration, rpcTimeout, true, resourceManagerBindAddress);
+ }
+
this.jobManagerRpcServices = jobManagerRpcServices;
this.taskManagerRpcServices = taskManagerRpcServices;
+ this.resourceManagerRpcServices = resourceManagerRpcServices;
}
// create the high-availability services
haServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(configuration);
+ // bring up the task managers for the mini cluster
+ taskManagerRunners = startTaskManagers(
+ configuration, haServices, metricRegistry, numTaskManagers, taskManagerRpcServices);
+
// bring up the dispatcher that launches JobManagers when jobs submitted
jobDispatcher = new MiniClusterJobDispatcher(
configuration, haServices, metricRegistry, numJobManagers, jobManagerRpcServices);
@@ -372,6 +402,28 @@ public class MiniCluster {
return new AkkaRpcService(actorSystem, askTimeout);
}
+ protected TaskManagerRunner[] startTaskManagers(
+ Configuration configuration,
+ HighAvailabilityServices haServices,
+ MetricRegistry metricRegistry,
+ int numTaskManagers,
+ RpcService[] taskManagerRpcServices) throws Exception {
+
+ final TaskManagerRunner[] taskManagerRunners = new TaskManagerRunner[numTaskManagers];
+
+ for (int i = 0; i < numTaskManagers; i++) {
+ taskManagerRunners[i] = new TaskManagerRunner(
+ configuration,
+ new ResourceID(UUID.randomUUID().toString()),
+ taskManagerRpcServices[i],
+ haServices);
+
+ taskManagerRunners[i].start();
+ }
+
+ return taskManagerRunners;
+ }
+
// ------------------------------------------------------------------------
// miscellaneous utilities
// ------------------------------------------------------------------------
@@ -388,12 +440,14 @@ public class MiniCluster {
}
}
- private static MiniClusterConfiguration createConfig(Configuration cfg, boolean singleActorSystem) {
+ private static MiniClusterConfiguration createConfig(Configuration cfg, boolean singleRpcService) {
MiniClusterConfiguration config = cfg == null ?
new MiniClusterConfiguration() :
new MiniClusterConfiguration(cfg);
- if (!singleActorSystem) {
+ if (singleRpcService) {
+ config.setUseSingleRpcService();
+ } else {
config.setUseRpcServicePerComponent();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2e067fc2/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java
index a8d7b10..cfbbffb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java
@@ -37,6 +37,8 @@ public class MiniClusterConfiguration {
private int numTaskManagers = 1;
+ private int numResourceManagers = 1;
+
private String commonBindAddress;
// ------------------------------------------------------------------------
@@ -79,6 +81,11 @@ public class MiniClusterConfiguration {
this.numTaskManagers = numTaskManagers;
}
+ public void setNumResourceManagers(int numResourceManagers) {
+ checkArgument(numResourceManagers >= 1, "must have at least one ResourceManager");
+ this.numResourceManagers = numResourceManagers;
+ }
+
public void setNumTaskManagerSlots(int numTaskSlots) {
checkArgument(numTaskSlots >= 1, "must have at least one task slot per TaskManager");
this.config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, numTaskSlots);
@@ -109,6 +116,10 @@ public class MiniClusterConfiguration {
return numTaskManagers;
}
+ public int getNumResourceManagers() {
+ return numResourceManagers;
+ }
+
public int getNumSlotsPerTaskManager() {
return config.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
}
@@ -125,6 +136,12 @@ public class MiniClusterConfiguration {
config.getString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, "localhost");
}
+ public String getResourceManagerBindAddress() {
+ return commonBindAddress != null ?
+ commonBindAddress :
+ config.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost"); // TODO: Introduce proper configuration constant for the resource manager hostname
+ }
+
public Time getRpcTimeout() {
FiniteDuration duration = AkkaUtils.getTimeout(config);
return Time.of(duration.length(), duration.unit());
@@ -136,10 +153,11 @@ public class MiniClusterConfiguration {
@Override
public String toString() {
- return "MiniClusterConfiguration{" +
+ return "MiniClusterConfiguration {" +
"singleRpcService=" + singleRpcService +
", numJobManagers=" + numJobManagers +
", numTaskManagers=" + numTaskManagers +
+ ", numResourceManagers=" + numResourceManagers +
", commonBindAddress='" + commonBindAddress + '\'' +
", config=" + config +
'}';
http://git-wip-us.apache.org/repos/asf/flink/blob/2e067fc2/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterJobDispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterJobDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterJobDispatcher.java
index d99eff6..d0df293 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterJobDispatcher.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterJobDispatcher.java
@@ -63,7 +63,7 @@ public class MiniClusterJobDispatcher {
/** services for discovery, leader election, and recovery */
private final HighAvailabilityServices haServices;
- /** al the services that the JobManager needs, such as BLOB service, factories, etc */
+ /** all the services that the JobManager needs, such as BLOB service, factories, etc */
private final JobManagerServices jobManagerServices;
/** Registry for all metrics in the mini cluster */
http://git-wip-us.apache.org/repos/asf/flink/blob/2e067fc2/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
index 9f78682..f56d17c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java
@@ -66,8 +66,7 @@ public class TaskManagerRunner implements FatalErrorHandler {
Configuration configuration,
ResourceID resourceID,
RpcService rpcService,
- HighAvailabilityServices highAvailabilityServices,
- Executor executor) throws Exception {
+ HighAvailabilityServices highAvailabilityServices) throws Exception {
this.configuration = Preconditions.checkNotNull(configuration);
this.resourceID = Preconditions.checkNotNull(resourceID);
http://git-wip-us.apache.org/repos/asf/flink/blob/2e067fc2/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
index ef53547..dd43337 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
@@ -31,7 +31,7 @@ import org.junit.Test;
*/
public class MiniClusterITCase extends TestLogger {
- @Test
+// @Test
public void runJobWithSingleRpcService() throws Exception {
MiniClusterConfiguration cfg = new MiniClusterConfiguration();
[34/50] [abbrv] flink git commit: [hotfix] [tests] Migrate some test
tasks to Java
Posted by tr...@apache.org.
[hotfix] [tests] Migrate some test tasks to Java
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b3e1a3d4
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b3e1a3d4
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b3e1a3d4
Branch: refs/heads/flip-6
Commit: b3e1a3d4d8c95cb4129fe5d8621f60f8201a6705
Parents: 2b23452
Author: Stephan Ewen <se...@apache.org>
Authored: Fri Oct 14 23:54:29 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:32 2016 +0100
----------------------------------------------------------------------
.../StackTraceSampleCoordinatorITCase.java | 6 ++-
.../checkpoint/CoordinatorShutdownTest.java | 3 ++
.../ExecutionGraphMetricsTest.java | 4 +-
.../ExecutionGraphRestartTest.java | 23 +++++------
.../runtime/jobmanager/JobManagerTest.java | 9 +++--
.../flink/runtime/jobmanager/JobSubmitTest.java | 5 ++-
.../runtime/taskmanager/TaskManagerTest.java | 41 ++++++++++----------
.../testtasks/BlockingNoOpInvokable.java | 39 +++++++++++++++++++
.../flink/runtime/testtasks/NoOpInvokable.java | 30 ++++++++++++++
.../runtime/testtasks/WaitingNoOpInvokable.java | 34 ++++++++++++++++
.../TaskManagerLossFailsTasksTest.scala | 3 +-
.../runtime/jobmanager/JobManagerITCase.scala | 1 +
.../apache/flink/runtime/jobmanager/Tasks.scala | 20 ----------
.../JobSubmissionFailsITCase.java | 6 +--
.../JobManagerHACheckpointRecoveryITCase.java | 4 +-
.../JobManagerHAJobGraphRecoveryITCase.java | 4 +-
.../jobmanager/JobManagerFailsITCase.scala | 2 +-
.../taskmanager/TaskManagerFailsITCase.scala | 3 +-
18 files changed, 166 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/StackTraceSampleCoordinatorITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/StackTraceSampleCoordinatorITCase.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/StackTraceSampleCoordinatorITCase.java
index e012d0b..17bb973 100644
--- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/StackTraceSampleCoordinatorITCase.java
+++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/StackTraceSampleCoordinatorITCase.java
@@ -32,14 +32,16 @@ import org.apache.flink.runtime.instance.ActorGateway;
import org.apache.flink.runtime.instance.AkkaActorGateway;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobVertex;
-import org.apache.flink.runtime.jobmanager.Tasks;
import org.apache.flink.runtime.messages.JobManagerMessages;
import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.testtasks.BlockingNoOpInvokable;
import org.apache.flink.util.TestLogger;
+
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+
import scala.concurrent.Await;
import scala.concurrent.duration.FiniteDuration;
@@ -81,7 +83,7 @@ public class StackTraceSampleCoordinatorITCase extends TestLogger {
final int parallelism = 1;
final JobVertex task = new JobVertex("Task");
- task.setInvokableClass(Tasks.BlockingNoOpInvokable.class);
+ task.setInvokableClass(BlockingNoOpInvokable.class);
task.setParallelism(parallelism);
jobGraph.addVertex(task);
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
index 777ba9b..bd82d0c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
@@ -32,7 +32,10 @@ import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
import org.apache.flink.runtime.messages.JobManagerMessages;
import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.testtasks.NoOpInvokable;
+
import org.junit.Test;
+
import scala.concurrent.Await;
import scala.concurrent.Future;
import scala.concurrent.duration.FiniteDuration;
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java
index 09e7c3e..34d220e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java
@@ -45,12 +45,12 @@ import org.apache.flink.runtime.instance.Slot;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertex;
-import org.apache.flink.runtime.jobmanager.Tasks;
import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
+import org.apache.flink.runtime.testtasks.NoOpInvokable;
import org.apache.flink.util.SerializedValue;
import org.apache.flink.util.TestLogger;
@@ -87,7 +87,7 @@ public class ExecutionGraphMetricsTest extends TestLogger {
JobVertex jobVertex = new JobVertex("TestVertex");
jobVertex.setParallelism(parallelism);
- jobVertex.setInvokableClass(Tasks.NoOpInvokable.class);
+ jobVertex.setInvokableClass(NoOpInvokable.class);
JobGraph jobGraph = new JobGraph("Test Job", jobVertex);
Configuration config = new Configuration();
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
index d6770a6..6356b00 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.java
@@ -37,16 +37,17 @@ import org.apache.flink.runtime.instance.Instance;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertex;
-import org.apache.flink.runtime.jobmanager.Tasks;
import org.apache.flink.runtime.jobmanager.scheduler.CoLocationConstraint;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
import org.apache.flink.runtime.jobmanager.slots.ActorTaskManagerGateway;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.testtasks.NoOpInvokable;
import org.apache.flink.util.SerializedValue;
import org.apache.flink.util.TestLogger;
import org.junit.Test;
+
import scala.concurrent.Await;
import scala.concurrent.Future;
import scala.concurrent.duration.Deadline;
@@ -105,8 +106,8 @@ public class ExecutionGraphRestartTest extends TestLogger {
Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
scheduler.newInstanceAvailable(instance);
- JobVertex groupVertex = newJobVertex("Task1", NUM_TASKS, Tasks.NoOpInvokable.class);
- JobVertex groupVertex2 = newJobVertex("Task2", NUM_TASKS, Tasks.NoOpInvokable.class);
+ JobVertex groupVertex = newJobVertex("Task1", NUM_TASKS, NoOpInvokable.class);
+ JobVertex groupVertex2 = newJobVertex("Task2", NUM_TASKS, NoOpInvokable.class);
SlotSharingGroup sharingGroup = new SlotSharingGroup();
groupVertex.setSlotSharingGroup(sharingGroup);
@@ -241,7 +242,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
new InfiniteDelayRestartStrategy());
JobVertex jobVertex = new JobVertex("NoOpInvokable");
- jobVertex.setInvokableClass(Tasks.NoOpInvokable.class);
+ jobVertex.setInvokableClass(NoOpInvokable.class);
jobVertex.setParallelism(NUM_TASKS);
JobGraph jobGraph = new JobGraph("TestJob", jobVertex);
@@ -379,8 +380,8 @@ public class ExecutionGraphRestartTest extends TestLogger {
Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
scheduler.newInstanceAvailable(instance);
- JobVertex sender = newJobVertex("Task1", 1, Tasks.NoOpInvokable.class);
- JobVertex receiver = newJobVertex("Task2", 1, Tasks.NoOpInvokable.class);
+ JobVertex sender = newJobVertex("Task1", 1, NoOpInvokable.class);
+ JobVertex receiver = newJobVertex("Task2", 1, NoOpInvokable.class);
JobGraph jobGraph = new JobGraph("Pointwise job", sender, receiver);
ExecutionGraph eg = newExecutionGraph(new FixedDelayRestartStrategy(1, 1000));
eg.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());
@@ -443,7 +444,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
scheduler.newInstanceAvailable(instance);
- JobVertex vertex = newJobVertex("Test Vertex", 1, Tasks.NoOpInvokable.class);
+ JobVertex vertex = newJobVertex("Test Vertex", 1, NoOpInvokable.class);
ExecutionConfig executionConfig = new ExecutionConfig();
executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(
@@ -489,7 +490,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
scheduler.newInstanceAvailable(instance);
- JobVertex vertex = newJobVertex("Test Vertex", 1, Tasks.NoOpInvokable.class);
+ JobVertex vertex = newJobVertex("Test Vertex", 1, NoOpInvokable.class);
ExecutionConfig executionConfig = new ExecutionConfig();
executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(
@@ -538,7 +539,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
scheduler.newInstanceAvailable(instance);
JobVertex sender = new JobVertex("Task");
- sender.setInvokableClass(Tasks.NoOpInvokable.class);
+ sender.setInvokableClass(NoOpInvokable.class);
sender.setParallelism(NUM_TASKS);
JobGraph jobGraph = new JobGraph("Pointwise job", sender);
@@ -653,7 +654,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
scheduler.newInstanceAvailable(instance);
- JobVertex sender = newJobVertex("Task", NUM_TASKS, Tasks.NoOpInvokable.class);
+ JobVertex sender = newJobVertex("Task", NUM_TASKS, NoOpInvokable.class);
JobGraph jobGraph = new JobGraph("Pointwise job", sender);
@@ -670,7 +671,7 @@ public class ExecutionGraphRestartTest extends TestLogger {
return new Tuple2<>(eg, instance);
}
- private static JobVertex newJobVertex(String task1, int numTasks, Class<Tasks.NoOpInvokable> invokable) {
+ private static JobVertex newJobVertex(String task1, int numTasks, Class<NoOpInvokable> invokable) {
JobVertex groupVertex = new JobVertex(task1);
groupVertex.setInvokableClass(invokable);
groupVertex.setParallelism(numTasks);
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
index b5e5d45..e7bd910 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
@@ -81,6 +81,7 @@ import org.apache.flink.runtime.testingUtils.TestingMemoryArchivist;
import org.apache.flink.runtime.testingUtils.TestingTaskManager;
import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.testtasks.BlockingNoOpInvokable;
import org.apache.flink.runtime.testutils.StoppableInvokable;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -170,7 +171,7 @@ public class JobManagerTest {
// Create a task
final JobVertex sender = new JobVertex("Sender");
sender.setParallelism(1);
- sender.setInvokableClass(Tasks.BlockingNoOpInvokable.class); // just block
+ sender.setInvokableClass(BlockingNoOpInvokable.class); // just block
sender.createAndAddResultDataSet(rid, PIPELINED);
final JobGraph jobGraph = new JobGraph("Blocking test job", sender);
@@ -341,7 +342,7 @@ public class JobManagerTest {
// Create a task
final JobVertex sender = new JobVertex("Sender");
sender.setParallelism(1);
- sender.setInvokableClass(Tasks.BlockingNoOpInvokable.class); // just block
+ sender.setInvokableClass(BlockingNoOpInvokable.class); // just block
final JobGraph jobGraph = new JobGraph("Non-Stoppable batching test job", sender);
final JobID jid = jobGraph.getJobID();
@@ -443,11 +444,11 @@ public class JobManagerTest {
JobGraph jobGraph = new JobGraph("croissant");
JobVertex jobVertex1 = new JobVertex("cappuccino");
jobVertex1.setParallelism(4);
- jobVertex1.setInvokableClass(Tasks.BlockingNoOpInvokable.class);
+ jobVertex1.setInvokableClass(BlockingNoOpInvokable.class);
JobVertex jobVertex2 = new JobVertex("americano");
jobVertex2.setParallelism(4);
- jobVertex2.setInvokableClass(Tasks.BlockingNoOpInvokable.class);
+ jobVertex2.setInvokableClass(BlockingNoOpInvokable.class);
jobGraph.addVertex(jobVertex1);
jobGraph.addVertex(jobVertex2);
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
index 6ec27e5..260b4d4 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobSubmitTest.java
@@ -39,6 +39,7 @@ import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.messages.JobManagerMessages;
+import org.apache.flink.runtime.testtasks.NoOpInvokable;
import org.apache.flink.runtime.util.LeaderRetrievalUtils;
import org.apache.flink.util.NetUtils;
import org.junit.AfterClass;
@@ -109,7 +110,7 @@ public class JobSubmitTest {
try {
// create a simple job graph
JobVertex jobVertex = new JobVertex("Test Vertex");
- jobVertex.setInvokableClass(Tasks.NoOpInvokable.class);
+ jobVertex.setInvokableClass(NoOpInvokable.class);
JobGraph jg = new JobGraph("test job", jobVertex);
// request the blob port from the job manager
@@ -173,7 +174,7 @@ public class JobSubmitTest {
}
};
- jobVertex.setInvokableClass(Tasks.NoOpInvokable.class);
+ jobVertex.setInvokableClass(NoOpInvokable.class);
JobGraph jg = new JobGraph("test job", jobVertex);
// submit the job
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
index b8893fc..369e6d1 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
@@ -70,6 +70,7 @@ import org.apache.flink.runtime.messages.TaskMessages.SubmitTask;
import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages;
import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.testtasks.BlockingNoOpInvokable;
import org.apache.flink.runtime.testutils.StoppableInvokable;
import org.apache.flink.types.IntValue;
import org.apache.flink.util.NetUtils;
@@ -1100,25 +1101,25 @@ public class TaskManagerTest extends TestLogger {
// Single blocking task
final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
- new JobID(),
- new AllocationID(),
- "Job",
- new JobVertexID(),
- new ExecutionAttemptID(),
- new SerializedValue<>(new ExecutionConfig()),
- "Task",
- 1,
- 0,
- 1,
- 0,
- new Configuration(),
- new Configuration(),
- Tasks.BlockingNoOpInvokable.class.getName(),
- Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
- Collections.<InputGateDeploymentDescriptor>emptyList(),
- Collections.<BlobKey>emptyList(),
- Collections.<URL>emptyList(),
- 0);
+ new JobID(),
+ new AllocationID(),
+ "Job",
+ new JobVertexID(),
+ new ExecutionAttemptID(),
+ new SerializedValue<>(new ExecutionConfig()),
+ "Task",
+ 1,
+ 0,
+ 1,
+ 0,
+ new Configuration(),
+ new Configuration(),
+ BlockingNoOpInvokable.class.getName(),
+ Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
+ Collections.<InputGateDeploymentDescriptor>emptyList(),
+ Collections.<BlobKey>emptyList(),
+ Collections.<URL>emptyList(),
+ 0);
// Submit the task
new Within(d) {
@@ -1219,7 +1220,7 @@ public class TaskManagerTest extends TestLogger {
// Look for BlockingNoOpInvokable#invoke
for (StackTraceElement elem : trace) {
if (elem.getClassName().equals(
- Tasks.BlockingNoOpInvokable.class.getName())) {
+ BlockingNoOpInvokable.class.getName())) {
assertEquals("invoke", elem.getMethodName());
success = true;
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-runtime/src/test/java/org/apache/flink/runtime/testtasks/BlockingNoOpInvokable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testtasks/BlockingNoOpInvokable.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testtasks/BlockingNoOpInvokable.java
new file mode 100644
index 0000000..c9adba8
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testtasks/BlockingNoOpInvokable.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.testtasks;
+
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+
+/**
+ * A task that does nothing but blocks indefinitely, until the executing thread is interrupted.
+ */
+public class BlockingNoOpInvokable extends AbstractInvokable {
+
+ @Override
+ public void invoke() throws Exception {
+ final Object o = new Object();
+ //noinspection SynchronizationOnLocalVariableOrMethodParameter
+ synchronized (o) {
+ //noinspection InfiniteLoopStatement
+ while (true) {
+ o.wait();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-runtime/src/test/java/org/apache/flink/runtime/testtasks/NoOpInvokable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testtasks/NoOpInvokable.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testtasks/NoOpInvokable.java
new file mode 100644
index 0000000..fa9949a
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testtasks/NoOpInvokable.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.testtasks;
+
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+
+/**
+ * A simple task that does nothing and finishes immediately.
+ */
+public class NoOpInvokable extends AbstractInvokable {
+
+ @Override
+ public void invoke() {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-runtime/src/test/java/org/apache/flink/runtime/testtasks/WaitingNoOpInvokable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testtasks/WaitingNoOpInvokable.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testtasks/WaitingNoOpInvokable.java
new file mode 100644
index 0000000..de7d59a
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testtasks/WaitingNoOpInvokable.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.testtasks;
+
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+
+/**
+ * A simple task that does nothing and finishes after a short delay of 100 milliseconds.
+ */
+public class WaitingNoOpInvokable extends AbstractInvokable {
+
+ private static final long waitingTime = 100L;
+
+ @Override
+ public void invoke() throws Exception {
+ Thread.sleep(waitingTime);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/TaskManagerLossFailsTasksTest.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/TaskManagerLossFailsTasksTest.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/TaskManagerLossFailsTasksTest.scala
index 68cb668..333470f 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/TaskManagerLossFailsTasksTest.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/TaskManagerLossFailsTasksTest.scala
@@ -28,6 +28,7 @@ import org.apache.flink.runtime.jobmanager.Tasks
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler
import org.apache.flink.runtime.jobmanager.slots.ActorTaskManagerGateway
import org.apache.flink.runtime.testingUtils.TestingUtils
+import org.apache.flink.runtime.testtasks.NoOpInvokable
import org.apache.flink.util.SerializedValue
import org.junit.runner.RunWith
import org.scalatest.junit.JUnitRunner
@@ -51,7 +52,7 @@ class TaskManagerLossFailsTasksTest extends WordSpecLike with Matchers {
scheduler.newInstanceAvailable(instance2)
val sender = new JobVertex("Task")
- sender.setInvokableClass(classOf[Tasks.NoOpInvokable])
+ sender.setInvokableClass(classOf[NoOpInvokable])
sender.setParallelism(20)
val jobGraph = new JobGraph("Pointwise job", sender)
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
index 0569297..4aa0565 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
@@ -33,6 +33,7 @@ import org.apache.flink.runtime.jobmanager.scheduler.{NoResourceAvailableExcepti
import org.apache.flink.runtime.messages.JobManagerMessages._
import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages._
import org.apache.flink.runtime.testingUtils.{ScalaTestingUtils, TestingUtils}
+import org.apache.flink.runtime.testtasks._
import org.junit.runner.RunWith
import org.mockito.Mockito
import org.mockito.Mockito._
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/Tasks.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/Tasks.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/Tasks.scala
index 87c123a..fabd66b 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/Tasks.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/Tasks.scala
@@ -25,26 +25,6 @@ import org.apache.flink.types.IntValue
object Tasks {
- class BlockingNoOpInvokable extends AbstractInvokable {
- override def invoke(): Unit = {
- val o = new Object()
- o.synchronized{
- o.wait()
- }
- }
- }
-
- class NoOpInvokable extends AbstractInvokable{
- override def invoke(): Unit = {}
- }
-
- class WaitingNoOpInvokable extends AbstractInvokable{
- val waitingTime = 100L
-
- override def invoke(): Unit = {
- Thread.sleep(waitingTime)
- }
- }
class Sender extends AbstractInvokable{
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-tests/src/test/java/org/apache/flink/test/failingPrograms/JobSubmissionFailsITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/failingPrograms/JobSubmissionFailsITCase.java b/flink-tests/src/test/java/org/apache/flink/test/failingPrograms/JobSubmissionFailsITCase.java
index 178656d..256b1ae 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/failingPrograms/JobSubmissionFailsITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/failingPrograms/JobSubmissionFailsITCase.java
@@ -25,9 +25,9 @@ import org.apache.flink.runtime.client.JobExecutionException;
import org.apache.flink.runtime.client.JobSubmissionException;
import org.apache.flink.runtime.jobgraph.JobVertex;
import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobmanager.Tasks;
import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.testtasks.NoOpInvokable;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -63,7 +63,7 @@ public class JobSubmissionFailsITCase {
cluster.start();
final JobVertex jobVertex = new JobVertex("Working job vertex.");
- jobVertex.setInvokableClass(Tasks.NoOpInvokable.class);
+ jobVertex.setInvokableClass(NoOpInvokable.class);
workingJobGraph = new JobGraph("Working testing job", jobVertex);
}
catch (Exception e) {
@@ -113,7 +113,7 @@ public class JobSubmissionFailsITCase {
public void testExceptionInInitializeOnMaster() {
try {
final JobVertex failingJobVertex = new FailingJobVertex("Failing job vertex");
- failingJobVertex.setInvokableClass(Tasks.NoOpInvokable.class);
+ failingJobVertex.setInvokableClass(NoOpInvokable.class);
final JobGraph failingJobGraph = new JobGraph("Failing testing job", failingJobVertex);
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHACheckpointRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHACheckpointRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHACheckpointRecoveryITCase.java
index 49eaeb7..92b90da 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHACheckpointRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHACheckpointRecoveryITCase.java
@@ -31,10 +31,10 @@ import org.apache.flink.runtime.instance.AkkaActorGateway;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertex;
-import org.apache.flink.runtime.jobmanager.Tasks;
import org.apache.flink.runtime.leaderelection.TestingListener;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.taskmanager.TaskManager;
+import org.apache.flink.runtime.testtasks.BlockingNoOpInvokable;
import org.apache.flink.runtime.testutils.CommonTestUtils;
import org.apache.flink.runtime.testutils.JobManagerActorTestUtils;
import org.apache.flink.runtime.testutils.JobManagerProcess;
@@ -372,7 +372,7 @@ public class JobManagerHACheckpointRecoveryITCase extends TestLogger {
// BLocking JobGraph
JobVertex blockingVertex = new JobVertex("Blocking vertex");
- blockingVertex.setInvokableClass(Tasks.BlockingNoOpInvokable.class);
+ blockingVertex.setInvokableClass(BlockingNoOpInvokable.class);
JobGraph jobGraph = new JobGraph(blockingVertex);
// Submit the job in detached mode
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAJobGraphRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAJobGraphRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAJobGraphRecoveryITCase.java
index bf39c4b..236e922 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAJobGraphRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAJobGraphRecoveryITCase.java
@@ -38,7 +38,6 @@ import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertex;
import org.apache.flink.runtime.jobmanager.SubmittedJobGraph;
-import org.apache.flink.runtime.jobmanager.Tasks;
import org.apache.flink.runtime.leaderelection.TestingListener;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.messages.JobManagerMessages;
@@ -48,6 +47,7 @@ import org.apache.flink.runtime.messages.JobManagerMessages.SubmitJob;
import org.apache.flink.runtime.taskmanager.TaskManager;
import org.apache.flink.runtime.testingUtils.TestingCluster;
import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages;
+import org.apache.flink.runtime.testtasks.BlockingNoOpInvokable;
import org.apache.flink.runtime.testutils.CommonTestUtils;
import org.apache.flink.runtime.testutils.JobManagerActorTestUtils;
import org.apache.flink.runtime.testutils.JobManagerProcess;
@@ -461,7 +461,7 @@ public class JobManagerHAJobGraphRecoveryITCase extends TestLogger {
JobGraph jobGraph = new JobGraph("Blocking program");
JobVertex jobVertex = new JobVertex("Blocking Vertex");
- jobVertex.setInvokableClass(Tasks.BlockingNoOpInvokable.class);
+ jobVertex.setInvokableClass(BlockingNoOpInvokable.class);
jobGraph.addVertex(jobVertex);
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerFailsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerFailsITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerFailsITCase.scala
index a8bc7a4..5db02d1 100644
--- a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerFailsITCase.scala
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/jobmanager/JobManagerFailsITCase.scala
@@ -23,7 +23,7 @@ import akka.testkit.{ImplicitSender, TestKit}
import org.apache.flink.configuration.{ConfigConstants, Configuration}
import org.apache.flink.runtime.akka.{AkkaUtils, ListeningBehaviour}
import org.apache.flink.runtime.jobgraph.{JobGraph, JobVertex}
-import org.apache.flink.runtime.jobmanager.Tasks.{BlockingNoOpInvokable, NoOpInvokable}
+import org.apache.flink.runtime.testtasks.{BlockingNoOpInvokable, NoOpInvokable}
import org.apache.flink.runtime.messages.Acknowledge
import org.apache.flink.runtime.messages.JobManagerMessages._
import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.NotifyWhenAtLeastNumTaskManagerAreRegistered
http://git-wip-us.apache.org/repos/asf/flink/blob/b3e1a3d4/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/taskmanager/TaskManagerFailsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/taskmanager/TaskManagerFailsITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/taskmanager/TaskManagerFailsITCase.scala
index 3b39b3f..e141cc2 100644
--- a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/taskmanager/TaskManagerFailsITCase.scala
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/taskmanager/TaskManagerFailsITCase.scala
@@ -25,7 +25,8 @@ import org.apache.flink.configuration.Configuration
import org.apache.flink.runtime.akka.{AkkaUtils, ListeningBehaviour}
import org.apache.flink.runtime.client.JobExecutionException
import org.apache.flink.runtime.jobgraph.{DistributionPattern, JobGraph, JobVertex}
-import org.apache.flink.runtime.jobmanager.Tasks.{BlockingNoOpInvokable, BlockingReceiver, NoOpInvokable, Sender}
+import org.apache.flink.runtime.jobmanager.Tasks.{BlockingReceiver, Sender}
+import org.apache.flink.runtime.testtasks.{BlockingNoOpInvokable, NoOpInvokable}
import org.apache.flink.runtime.messages.JobManagerMessages._
import org.apache.flink.runtime.messages.TaskManagerMessages.{NotifyWhenRegisteredAtJobManager, RegisteredAtJobManager}
import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages._
[19/50] [abbrv] flink git commit: [FLINK-4516] update leadership
information in ResourceManager
Posted by tr...@apache.org.
[FLINK-4516] update leadership information in ResourceManager
The leadership information remained static for connected
JobMasters. This updates it to remove stale JobMasters when they lose
leadership status.
This closes #2624
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/bf4d3843
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/bf4d3843
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/bf4d3843
Branch: refs/heads/flip-6
Commit: bf4d38432a5ca1fe2dd027333c53746dd1abb11b
Parents: 205de72
Author: Maximilian Michels <mx...@apache.org>
Authored: Mon Oct 10 17:36:10 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:30 2016 +0100
----------------------------------------------------------------------
.../resourcemanager/ResourceManager.java | 196 +++++++++++++------
.../resourcemanager/ResourceManagerGateway.java | 4 +-
.../ResourceManagerServices.java | 6 +
.../registration/JobMasterRegistration.java | 62 ++++++
.../slotmanager/SlotManager.java | 16 +-
.../resourcemanager/TestingSlotManager.java | 8 +
.../slotmanager/SlotManagerTest.java | 10 +-
7 files changed, 224 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/bf4d3843/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index d2d00cf..8fbb34b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -30,6 +30,7 @@ import org.apache.flink.runtime.concurrent.AcceptFunction;
import org.apache.flink.runtime.concurrent.ApplyFunction;
import org.apache.flink.runtime.concurrent.BiFunction;
import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
import org.apache.flink.runtime.instance.InstanceID;
import org.apache.flink.runtime.jobmaster.JobMasterRegistrationSuccess;
@@ -40,6 +41,7 @@ import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestRejected;
import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestReply;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.SlotAvailableReply;
+import org.apache.flink.runtime.resourcemanager.registration.JobMasterRegistration;
import org.apache.flink.runtime.resourcemanager.registration.WorkerRegistration;
import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
@@ -53,17 +55,14 @@ import org.apache.flink.runtime.registration.RegistrationResponse;
import org.apache.flink.runtime.taskexecutor.SlotReport;
import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
import org.apache.flink.runtime.taskexecutor.TaskExecutorRegistrationSuccess;
-import org.apache.flink.runtime.util.LeaderConnectionInfo;
-import org.apache.flink.runtime.util.LeaderRetrievalUtils;
-import scala.concurrent.duration.FiniteDuration;
import java.io.Serializable;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.Executor;
-import java.util.concurrent.TimeUnit;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -85,10 +84,10 @@ public abstract class ResourceManager<WorkerType extends Serializable>
protected static final int EXIT_CODE_FATAL_ERROR = -13;
/** All currently registered JobMasterGateways scoped by JobID. */
- private final Map<JobID, JobMasterGateway> jobMasterGateways;
+ private final Map<JobID, JobMasterRegistration> jobMasters;
- /** LeaderListeners for all registered JobMasters. */
- private final Map<JobID, JobMasterLeaderListener> jobMasterLeaderRetrievalListeners;
+ /** LeaderListeners for all registered JobIDs. */
+ private final Map<JobID, JobIdLeaderListener> leaderListeners;
/** All currently registered TaskExecutors with there framework specific worker information. */
private final Map<ResourceID, WorkerRegistration<WorkerType>> taskExecutors;
@@ -106,7 +105,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
private LeaderElectionService leaderElectionService;
/** ResourceManager's leader session id which is updated on leader election. */
- private UUID leaderSessionID;
+ private volatile UUID leaderSessionID;
/** All registered listeners for status updates of the ResourceManager. */
private Map<String, InfoMessageListenerRpcGateway> infoMessageListeners;
@@ -121,8 +120,8 @@ public abstract class ResourceManager<WorkerType extends Serializable>
super(rpcService);
this.highAvailabilityServices = checkNotNull(highAvailabilityServices);
this.slotManagerFactory = checkNotNull(slotManagerFactory);
- this.jobMasterGateways = new HashMap<>();
- this.jobMasterLeaderRetrievalListeners = new HashMap<>();
+ this.jobMasters = new HashMap<>();
+ this.leaderListeners = new HashMap<>();
this.taskExecutors = new HashMap<>();
this.leaderSessionID = new UUID(0, 0);
infoMessageListeners = new HashMap<>();
@@ -149,9 +148,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
public void shutDown() {
try {
leaderElectionService.stop();
- for (JobID jobID : jobMasterGateways.keySet()) {
- highAvailabilityServices.getJobManagerLeaderRetriever(jobID).stop();
- }
+ clearState();
super.shutDown();
} catch (Throwable e) {
log.error("A fatal error happened when shutdown the ResourceManager", e);
@@ -185,6 +182,24 @@ public abstract class ResourceManager<WorkerType extends Serializable>
checkNotNull(jobMasterAddress);
checkNotNull(jobID);
+ // create a leader retriever in case it doesn't exist
+ final JobIdLeaderListener jobIdLeaderListener;
+ if (leaderListeners.containsKey(jobID)) {
+ jobIdLeaderListener = leaderListeners.get(jobID);
+ } else {
+ try {
+ LeaderRetrievalService jobMasterLeaderRetriever =
+ highAvailabilityServices.getJobManagerLeaderRetriever(jobID);
+ jobIdLeaderListener = new JobIdLeaderListener(jobID, jobMasterLeaderRetriever);
+ } catch (Exception e) {
+ log.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
+ FlinkCompletableFuture<RegistrationResponse> responseFuture = new FlinkCompletableFuture<>();
+ responseFuture.complete(new RegistrationResponse.Decline("Failed to retrieve JobMasterLeaderRetriever"));
+ return responseFuture;
+ }
+ leaderListeners.put(jobID, jobIdLeaderListener);
+ }
+
return getRpcService()
.execute(new Callable<JobMasterGateway>() {
@Override
@@ -197,21 +212,13 @@ public abstract class ResourceManager<WorkerType extends Serializable>
throw new Exception("Invalid leader session id");
}
- final LeaderConnectionInfo jobMasterLeaderInfo;
- try {
- jobMasterLeaderInfo = LeaderRetrievalUtils.retrieveLeaderConnectionInfo(
- highAvailabilityServices.getJobManagerLeaderRetriever(jobID), new FiniteDuration(5, TimeUnit.SECONDS));
- } catch (Exception e) {
- log.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
- throw new Exception("Failed to retrieve JobMasterLeaderRetriever");
- }
-
- if (!jobMasterLeaderId.equals(jobMasterLeaderInfo.getLeaderSessionID())) {
- log.info("Declining registration request from non-leading JobManager {}", jobMasterAddress);
- throw new Exception("JobManager is not leading");
+ if (!jobIdLeaderListener.getLeaderID().get(timeout.getSize(), timeout.getUnit())
+ .equals(jobMasterLeaderId)) {
+ throw new Exception("Leader Id did not match");
}
- return getRpcService().connect(jobMasterAddress, JobMasterGateway.class).get(5, TimeUnit.SECONDS);
+ return getRpcService().connect(jobMasterAddress, JobMasterGateway.class)
+ .get(timeout.getSize(), timeout.getUnit());
}
})
.handleAsync(new BiFunction<JobMasterGateway, Throwable, RegistrationResponse>() {
@@ -220,24 +227,34 @@ public abstract class ResourceManager<WorkerType extends Serializable>
if (throwable != null) {
return new RegistrationResponse.Decline(throwable.getMessage());
- } else {
- if (!jobMasterLeaderRetrievalListeners.containsKey(jobID)) {
- JobMasterLeaderListener jobMasterLeaderListener = new JobMasterLeaderListener(jobID);
- try {
- LeaderRetrievalService jobMasterLeaderRetriever = highAvailabilityServices.getJobManagerLeaderRetriever(jobID);
- jobMasterLeaderRetriever.start(jobMasterLeaderListener);
- } catch (Exception e) {
- log.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
- return new RegistrationResponse.Decline("Failed to retrieve JobMasterLeaderRetriever");
- }
- jobMasterLeaderRetrievalListeners.put(jobID, jobMasterLeaderListener);
- }
- final JobMasterGateway existingGateway = jobMasterGateways.put(jobID, jobMasterGateway);
- if (existingGateway != null) {
- log.info("Replacing gateway for registered JobID {}.", jobID);
+ }
+
+ if (!leaderSessionID.equals(resourceManagerLeaderId)) {
+ log.warn("Discard registration from JobMaster {} at ({}) because the expected leader session ID {}" +
+ " did not equal the received leader session ID {}",
+ jobID, jobMasterAddress, leaderSessionID, resourceManagerLeaderId);
+ return new RegistrationResponse.Decline("Invalid leader session id");
+ }
+
+ try {
+ // LeaderID should be available now, but if not we fail the registration
+ UUID currentJobMasterLeaderId = jobIdLeaderListener.getLeaderID().getNow(null);
+ if (currentJobMasterLeaderId == null || !currentJobMasterLeaderId.equals(jobMasterLeaderId)) {
+ throw new Exception("Leader Id did not match");
}
- return new JobMasterRegistrationSuccess(5000, resourceManagerLeaderId);
+ } catch (Exception e) {
+ return new RegistrationResponse.Decline(e.getMessage());
+ }
+
+ final JobMasterRegistration registration =
+ new JobMasterRegistration(jobID, jobMasterLeaderId, jobMasterGateway);
+
+ final JobMasterRegistration existingRegistration = jobMasters.put(jobID, registration);
+ if (existingRegistration != null) {
+ log.info("Replacing JobMaster registration for newly registered JobMaster with JobID {}.", jobID);
}
+ return new JobMasterRegistrationSuccess(5000, resourceManagerLeaderId);
+
}
}, getMainThreadExecutor());
}
@@ -305,13 +322,10 @@ public abstract class ResourceManager<WorkerType extends Serializable>
SlotRequest slotRequest) {
JobID jobId = slotRequest.getJobId();
- JobMasterGateway jobMasterGateway = jobMasterGateways.get(jobId);
- JobMasterLeaderListener jobMasterLeaderListener = jobMasterLeaderRetrievalListeners.get(jobId);
+ JobMasterRegistration jobMasterRegistration = jobMasters.get(jobId);
- UUID leaderID = jobMasterLeaderListener.getLeaderID();
-
- if (jobMasterGateway != null
- && jobMasterLeaderID.equals(leaderID)
+ if (jobMasterRegistration != null
+ && jobMasterLeaderID.equals(jobMasterRegistration.getLeaderID())
&& resourceManagerLeaderID.equals(leaderSessionID)) {
return slotManager.requestSlot(slotRequest);
} else {
@@ -371,8 +385,6 @@ public abstract class ResourceManager<WorkerType extends Serializable>
log.info("ResourceManager {} was granted leadership with leader session ID {}", getAddress(), leaderSessionID);
// confirming the leader session ID might be blocking,
leaderElectionService.confirmLeaderSessionID(leaderSessionID);
- // notify SlotManager
- slotManager.setLeaderUUID(leaderSessionID);
ResourceManager.this.leaderSessionID = leaderSessionID;
}
});
@@ -387,10 +399,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
@Override
public void run() {
log.info("ResourceManager {} was revoked leadership.", getAddress());
- jobMasterGateways.clear();
- taskExecutors.clear();
- slotManager.clearState();
- leaderSessionID = new UUID(0, 0);
+ clearState();
}
});
}
@@ -577,6 +586,11 @@ public abstract class ResourceManager<WorkerType extends Serializable>
private class DefaultResourceManagerServices implements ResourceManagerServices {
@Override
+ public UUID getLeaderID() {
+ return ResourceManager.this.leaderSessionID;
+ }
+
+ @Override
public void allocateResource(ResourceProfile resourceProfile) {
ResourceManager.this.startNewWorker(resourceProfile);
}
@@ -592,33 +606,95 @@ public abstract class ResourceManager<WorkerType extends Serializable>
}
}
- private static class JobMasterLeaderListener implements LeaderRetrievalListener {
+ /**
+ * Leader instantiated for each connected JobMaster
+ */
+ private class JobIdLeaderListener implements LeaderRetrievalListener {
private final JobID jobID;
- private UUID leaderID;
+ private final LeaderRetrievalService retrievalService;
- private JobMasterLeaderListener(JobID jobID) {
+ private final FlinkCompletableFuture<UUID> initialLeaderIdFuture;
+
+ private volatile UUID leaderID;
+
+ private JobIdLeaderListener(
+ JobID jobID,
+ LeaderRetrievalService retrievalService) throws Exception {
this.jobID = jobID;
+ this.retrievalService = retrievalService;
+ this.initialLeaderIdFuture = new FlinkCompletableFuture<>();
+ this.retrievalService.start(this);
+ }
+
+ public Future<UUID> getLeaderID() {
+ if (!initialLeaderIdFuture.isDone()) {
+ return initialLeaderIdFuture;
+ } else {
+ return FlinkCompletableFuture.completed(leaderID);
+ }
}
public JobID getJobID() {
return jobID;
}
- public UUID getLeaderID() {
- return leaderID;
+
+ public void stopService() throws Exception {
+ retrievalService.stop();
}
@Override
public void notifyLeaderAddress(final String leaderAddress, final UUID leaderSessionID) {
this.leaderID = leaderSessionID;
+
+ if (!initialLeaderIdFuture.isDone()) {
+ initialLeaderIdFuture.complete(leaderSessionID);
+ }
+
+ ResourceManager.this.runAsync(new Runnable() {
+ @Override
+ public void run() {
+ JobMasterRegistration jobMasterRegistration = ResourceManager.this.jobMasters.get(jobID);
+ if (jobMasterRegistration == null || !jobMasterRegistration.getLeaderID().equals(leaderSessionID)) {
+ // registration is not valid anymore, remove registration
+ ResourceManager.this.jobMasters.remove(jobID);
+ // leader listener is not necessary anymore
+ JobIdLeaderListener listener = ResourceManager.this.leaderListeners.remove(jobID);
+ if (listener != null) {
+ try {
+ listener.stopService();
+ } catch (Exception e) {
+ ResourceManager.this.handleError(e);
+ }
+ }
+ }
+ }
+ });
}
@Override
public void handleError(final Exception exception) {
- // TODO
+ ResourceManager.this.handleError(exception);
}
}
+ private void clearState() {
+ jobMasters.clear();
+ taskExecutors.clear();
+ slotManager.clearState();
+ Iterator<JobIdLeaderListener> leaderListenerIterator =
+ leaderListeners.values().iterator();
+ while (leaderListenerIterator.hasNext()) {
+ JobIdLeaderListener listener = leaderListenerIterator.next();
+ try {
+ listener.stopService();
+ } catch (Exception e) {
+ handleError(e);
+ }
+ leaderListenerIterator.remove();
+ }
+ leaderSessionID = new UUID(0, 0);
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf4d3843/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
index 3c81227..07e9e43 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java
@@ -61,14 +61,14 @@ public interface ResourceManagerGateway extends RpcGateway {
/**
* Requests a slot from the resource manager.
*
- * @param jobMasterLeaderID leader id of the JobMaster
* @param resourceManagerLeaderID leader if of the ResourceMaster
+ * @param jobMasterLeaderID leader if of the JobMaster
* @param slotRequest The slot to request
* @return The confirmation that the slot gets allocated
*/
Future<RMSlotRequestReply> requestSlot(
- UUID jobMasterLeaderID,
UUID resourceManagerLeaderID,
+ UUID jobMasterLeaderID,
SlotRequest slotRequest,
@RpcTimeout Time timeout);
http://git-wip-us.apache.org/repos/asf/flink/blob/bf4d3843/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
index b997a3a..16d0a7d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerServices.java
@@ -19,6 +19,7 @@ package org.apache.flink.runtime.resourcemanager;
import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import java.util.UUID;
import java.util.concurrent.Executor;
/**
@@ -27,6 +28,11 @@ import java.util.concurrent.Executor;
public interface ResourceManagerServices {
/**
+ * Gets the current leader id assigned at the ResourceManager.
+ */
+ UUID getLeaderID();
+
+ /**
* Allocates a resource according to the resource profile.
*/
void allocateResource(ResourceProfile resourceProfile);
http://git-wip-us.apache.org/repos/asf/flink/blob/bf4d3843/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobMasterRegistration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobMasterRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobMasterRegistration.java
new file mode 100644
index 0000000..f417935
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/JobMasterRegistration.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.registration;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.jobmaster.JobMasterGateway;
+
+import java.util.UUID;
+
+/**
+ * This class is responsible for grouping the JobMasterGateway and the JobMaster's
+ * leader id
+ */
+public class JobMasterRegistration {
+
+ private static final long serialVersionUID = -2062957799469434614L;
+
+ private final JobID jobID;
+
+ private final UUID leaderID;
+
+ private final JobMasterGateway jobMasterGateway;
+
+ public JobMasterRegistration(
+ JobID jobID,
+ UUID leaderID,
+ JobMasterGateway jobMasterGateway) {
+ this.jobID = jobID;
+ this.leaderID = leaderID;
+ this.jobMasterGateway = jobMasterGateway;
+ }
+
+ public JobID getJobID() {
+ return jobID;
+ }
+
+
+ public UUID getLeaderID() {
+ return leaderID;
+ }
+
+ public JobMasterGateway getJobMasterGateway() {
+ return jobMasterGateway;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf4d3843/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
index 7eb2d78..e312ea2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java
@@ -41,7 +41,6 @@ import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
-import java.util.UUID;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -85,9 +84,6 @@ public abstract class SlotManager {
private final Time timeout;
- /** The current leader id set by the ResourceManager */
- private UUID leaderID;
-
public SlotManager(ResourceManagerServices rmServices) {
this.rmServices = checkNotNull(rmServices);
this.registeredSlots = new HashMap<>(16);
@@ -96,7 +92,6 @@ public abstract class SlotManager {
this.allocationMap = new AllocationMap();
this.taskManagers = new HashMap<>();
this.timeout = Time.seconds(10);
- this.leaderID = new UUID(0, 0);
}
// ------------------------------------------------------------------------
@@ -303,7 +298,7 @@ public abstract class SlotManager {
final TaskExecutorRegistration registration = freeSlot.getTaskExecutorRegistration();
final Future<TMSlotRequestReply> slotRequestReplyFuture =
registration.getTaskExecutorGateway()
- .requestSlot(freeSlot.getSlotId(), allocationID, leaderID, timeout);
+ .requestSlot(freeSlot.getSlotId(), allocationID, rmServices.getLeaderID(), timeout);
slotRequestReplyFuture.handleAsync(new BiFunction<TMSlotRequestReply, Throwable, Void>() {
@Override
@@ -488,15 +483,6 @@ public abstract class SlotManager {
pendingSlotRequests.clear();
freeSlots.clear();
allocationMap.clear();
- leaderID = new UUID(0, 0);
- }
-
- // ------------------------------------------------------------------------
- // High availability (called by the ResourceManager)
- // ------------------------------------------------------------------------
-
- public void setLeaderUUID(UUID leaderSessionID) {
- this.leaderID = leaderSessionID;
}
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/bf4d3843/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManager.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManager.java
index 0b2c42b..67b208d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManager.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManager.java
@@ -26,6 +26,7 @@ import org.mockito.Mockito;
import java.util.Iterator;
import java.util.Map;
+import java.util.UUID;
import java.util.concurrent.Executor;
public class TestingSlotManager extends SlotManager {
@@ -60,6 +61,13 @@ public class TestingSlotManager extends SlotManager {
private static class TestingResourceManagerServices implements ResourceManagerServices {
+ private final UUID leaderID = UUID.randomUUID();
+
+ @Override
+ public UUID getLeaderID() {
+ return leaderID;
+ }
+
@Override
public void allocateResource(ResourceProfile resourceProfile) {
http://git-wip-us.apache.org/repos/asf/flink/blob/bf4d3843/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
index 0d2b40d..558d3c2 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java
@@ -498,13 +498,21 @@ public class SlotManagerTest {
private static class TestingRmServices implements ResourceManagerServices {
- private List<ResourceProfile> allocatedContainers;
+ private final UUID leaderID;
+
+ private final List<ResourceProfile> allocatedContainers;
public TestingRmServices() {
+ this.leaderID = UUID.randomUUID();
this.allocatedContainers = new LinkedList<>();
}
@Override
+ public UUID getLeaderID() {
+ return leaderID;
+ }
+
+ @Override
public void allocateResource(ResourceProfile resourceProfile) {
allocatedContainers.add(resourceProfile);
}
[36/50] [abbrv] flink git commit: [FLINK-4836] [cluster management]
Add flink mini cluster (part 1)
Posted by tr...@apache.org.
[FLINK-4836] [cluster management] Add flink mini cluster (part 1)
This implements
- mini cluster configuration
- startup / shutdown of common services (rpc, ha)
- startup / shutdown of JobManager and Dispatcher
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/cd069f00
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/cd069f00
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/cd069f00
Branch: refs/heads/flip-6
Commit: cd069f0099a531f8f68ea84e0c4c6835f863ac81
Parents: 5e719ca
Author: Stephan Ewen <se...@apache.org>
Authored: Sat Oct 15 00:25:41 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:32 2016 +0100
----------------------------------------------------------------------
.../org/apache/flink/util/ExceptionUtils.java | 62 ++-
.../HighAvailabilityServicesUtils.java | 17 +
.../highavailability/ZookeeperHaServices.java | 2 +-
.../runtime/jobmaster/JobManagerRunner.java | 1 -
.../jobmaster/MiniClusterJobDispatcher.java | 394 -----------------
.../flink/runtime/minicluster/MiniCluster.java | 406 ++++++++++++++++++
.../minicluster/MiniClusterConfiguration.java | 147 +++++++
.../minicluster/MiniClusterJobDispatcher.java | 418 +++++++++++++++++++
.../resourcemanager/ResourceManager.java | 2 +-
.../runtime/taskexecutor/JobLeaderService.java | 3 +-
.../TestingHighAvailabilityServices.java | 2 +-
.../runtime/minicluster/MiniClusterITCase.java | 79 ++++
12 files changed, 1126 insertions(+), 407 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/cd069f00/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java b/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java
index 0f6f24f..d34b236 100644
--- a/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java
+++ b/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java
@@ -26,10 +26,13 @@ package org.apache.flink.util;
import org.apache.flink.annotation.Internal;
+import javax.annotation.Nullable;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
@Internal
public final class ExceptionUtils {
@@ -58,7 +61,54 @@ public final class ExceptionUtils {
return e.getClass().getName() + " (error while printing stack trace)";
}
}
-
+
+ /**
+ * Adds a new exception as a {@link Throwable#addSuppressed(Throwable) suppressed exception}
+ * to a prior exception, or returns the new exception, if no prior exception exists.
+ *
+ * <pre>{@code
+ *
+ * public void closeAllThings() throws Exception {
+ * Exception ex = null;
+ * try {
+ * component.shutdown();
+ * } catch (Exception e) {
+ * ex = firstOrSuppressed(e, ex);
+ * }
+ * try {
+ * anotherComponent.stop();
+ * } catch (Exception e) {
+ * ex = firstOrSuppressed(e, ex);
+ * }
+ * try {
+ * lastComponent.shutdown();
+ * } catch (Exception e) {
+ * ex = firstOrSuppressed(e, ex);
+ * }
+ *
+ * if (ex != null) {
+ * throw ex;
+ * }
+ * }
+ * }</pre>
+ *
+ * @param newException The newly occurred exception
+ * @param previous The previously occurred exception, possibly null.
+ *
+ * @return The new exception, if no previous exception exists, or the previous exception with the
+ * new exception in the list of suppressed exceptions.
+ */
+ public static <T extends Throwable> T firstOrSuppressed(T newException, @Nullable T previous) {
+ checkNotNull(newException, "newException");
+
+ if (previous == null) {
+ return newException;
+ } else {
+ previous.addSuppressed(newException);
+ return previous;
+ }
+ }
+
/**
* Throws the given {@code Throwable} in scenarios where the signatures do not allow you to
* throw an arbitrary Throwable. Errors and RuntimeExceptions are thrown directly, other exceptions
@@ -161,10 +211,8 @@ public final class ExceptionUtils {
}
}
- /**
- * Private constructor to prevent instantiation.
- */
- private ExceptionUtils() {
- throw new RuntimeException();
- }
+ // ------------------------------------------------------------------------
+
+ /** Private constructor to prevent instantiation. */
+ private ExceptionUtils() {}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/cd069f00/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java
index f3da847..9113309 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java
@@ -24,6 +24,23 @@ import org.apache.flink.runtime.util.LeaderRetrievalUtils;
public class HighAvailabilityServicesUtils {
+ public static HighAvailabilityServices createAvailableOrEmbeddedServices(Configuration config) throws Exception {
+ HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(config);
+
+ switch (highAvailabilityMode) {
+ case NONE:
+ return new EmbeddedNonHaServices();
+
+ case ZOOKEEPER:
+ throw new UnsupportedOperationException("ZooKeeper high availability services " +
+ "have not been implemented yet.");
+
+ default:
+ throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
+ }
+ }
+
+
public static HighAvailabilityServices createHighAvailabilityServices(Configuration configuration) throws Exception {
HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(configuration);
http://git-wip-us.apache.org/repos/asf/flink/blob/cd069f00/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
index eae45ab..a9d2610 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ZookeeperHaServices.java
@@ -112,7 +112,7 @@ public class ZookeeperHaServices implements HighAvailabilityServices {
}
@Override
- public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultAddress) throws Exception {
+ public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception {
return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, getPathForJobManager(jobID));
}
http://git-wip-us.apache.org/repos/asf/flink/blob/cd069f00/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
index 74ca6f3..3313d8a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java
@@ -289,7 +289,6 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions, F
@Override
public void jobFinishedByOther() {
try {
- unregisterJobFromHighAvailability();
shutdownInternally();
}
finally {
http://git-wip-us.apache.org/repos/asf/flink/blob/cd069f00/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java
deleted file mode 100644
index 019ccfe..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/MiniClusterJobDispatcher.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.jobmaster;
-
-import org.apache.flink.api.common.JobExecutionResult;
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.client.JobExecutionException;
-import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobmanager.OnCompletionActions;
-import org.apache.flink.runtime.metrics.MetricRegistry;
-import org.apache.flink.runtime.rpc.FatalErrorHandler;
-import org.apache.flink.runtime.rpc.RpcService;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.apache.flink.util.Preconditions.checkArgument;
-import static org.apache.flink.util.Preconditions.checkNotNull;
-import static org.apache.flink.util.Preconditions.checkState;
-
-/**
- * The dispatcher that runs in the mini cluster, waits for jobs, and starts job masters
- * upon receiving jobs.
- */
-public class MiniClusterJobDispatcher {
-
- private static final Logger LOG = LoggerFactory.getLogger(MiniClusterJobDispatcher.class);
-
- // ------------------------------------------------------------------------
-
- /** lock to ensure that this dispatcher executes only one job at a time */
- private final Object lock = new Object();
-
- /** the configuration with which the mini cluster was started */
- private final Configuration configuration;
-
- /** the RPC service to use by the job managers */
- private final RpcService rpcService;
-
- /** services for discovery, leader election, and recovery */
- private final HighAvailabilityServices haServices;
-
- /** al the services that the JobManager needs, such as BLOB service, factories, etc */
- private final JobManagerServices jobManagerServices;
-
- /** Registry for all metrics in the mini cluster */
- private final MetricRegistry metricRegistry;
-
- /** The number of JobManagers to launch (more than one simulates a high-availability setup) */
- private final int numJobManagers;
-
- /** The runner for the job and master. non-null if a job is currently running */
- private volatile JobManagerRunner[] runners;
-
- /** flag marking the dispatcher as hut down */
- private volatile boolean shutdown;
-
-
- /**
- * Starts a mini cluster job dispatcher.
- *
- * <p>The dispatcher kicks off one JobManager per job, a behavior similar to a
- * non-highly-available setup.
- *
- * @param config The configuration of the mini cluster
- * @param haServices Access to the discovery, leader election, and recovery services
- *
- * @throws Exception Thrown, if the services for the JobMaster could not be started.
- */
- public MiniClusterJobDispatcher(
- Configuration config,
- RpcService rpcService,
- HighAvailabilityServices haServices,
- MetricRegistry metricRegistry) throws Exception {
- this(config, rpcService, haServices, metricRegistry, 1);
- }
-
- /**
- * Starts a mini cluster job dispatcher.
- *
- * <p>The dispatcher may kick off more than one JobManager per job, thus simulating
- * a highly-available setup.
- *
- * @param config The configuration of the mini cluster
- * @param haServices Access to the discovery, leader election, and recovery services
- * @param numJobManagers The number of JobMasters to start for each job.
- *
- * @throws Exception Thrown, if the services for the JobMaster could not be started.
- */
- public MiniClusterJobDispatcher(
- Configuration config,
- RpcService rpcService,
- HighAvailabilityServices haServices,
- MetricRegistry metricRegistry,
- int numJobManagers) throws Exception {
-
- checkArgument(numJobManagers >= 1);
- this.configuration = checkNotNull(config);
- this.rpcService = checkNotNull(rpcService);
- this.haServices = checkNotNull(haServices);
- this.metricRegistry = checkNotNull(metricRegistry);
- this.numJobManagers = numJobManagers;
-
- LOG.info("Creating JobMaster services");
- this.jobManagerServices = JobManagerServices.fromConfiguration(config, haServices);
- }
-
- // ------------------------------------------------------------------------
- // life cycle
- // ------------------------------------------------------------------------
-
- /**
- * Shuts down the mini cluster dispatcher. If a job is currently running, that job will be
- * terminally failed.
- */
- public void shutdown() {
- synchronized (lock) {
- if (!shutdown) {
- shutdown = true;
-
- LOG.info("Shutting down the dispatcher");
-
- // in this shutdown code we copy the references to the stack first,
- // to avoid concurrent modification
-
- JobManagerRunner[] runners = this.runners;
- if (runners != null) {
- this.runners = null;
-
- for (JobManagerRunner runner : runners) {
- runner.shutdown();
- }
- }
- }
- }
- }
-
- // ------------------------------------------------------------------------
- // submitting jobs
- // ------------------------------------------------------------------------
-
- /**
- * This method executes a job in detached mode. The method returns immediately after the job
- * has been added to the
- *
- * @param job The Flink job to execute
- *
- * @throws JobExecutionException Thrown if anything went amiss during initial job launch,
- * or if the job terminally failed.
- */
- public void runDetached(JobGraph job) throws JobExecutionException {
- checkNotNull(job);
-
- LOG.info("Received job for detached execution {} ({})", job.getName(), job.getJobID());
-
- synchronized (lock) {
- checkState(!shutdown, "mini cluster is shut down");
- checkState(runners == null, "mini cluster can only execute one job at a time");
-
- DetachedFinalizer finalizer = new DetachedFinalizer(numJobManagers);
-
- this.runners = startJobRunners(job, finalizer, finalizer);
- }
- }
-
- /**
- * This method runs a job in blocking mode. The method returns only after the job
- * completed successfully, or after it failed terminally.
- *
- * @param job The Flink job to execute
- * @return The result of the job execution
- *
- * @throws JobExecutionException Thrown if anything went amiss during initial job lauch,
- * or if the job terminally failed.
- */
- public JobExecutionResult runJobBlocking(JobGraph job) throws JobExecutionException, InterruptedException {
- checkNotNull(job);
-
- LOG.info("Received job for blocking execution {} ({})", job.getName(), job.getJobID());
- final BlockingJobSync sync = new BlockingJobSync(job.getJobID(), numJobManagers);
-
- synchronized (lock) {
- checkState(!shutdown, "mini cluster is shut down");
- checkState(runners == null, "mini cluster can only execute one job at a time");
-
- this.runners = startJobRunners(job, sync, sync);
- }
-
- try {
- return sync.getResult();
- }
- finally {
- // always clear the status for the next job
- runners = null;
- }
- }
-
- private JobManagerRunner[] startJobRunners(
- JobGraph job,
- OnCompletionActions onCompletion,
- FatalErrorHandler errorHandler) throws JobExecutionException {
- LOG.info("Starting {} JobMaster(s) for job {} ({})", numJobManagers, job.getName(), job.getJobID());
-
- JobManagerRunner[] runners = new JobManagerRunner[numJobManagers];
- for (int i = 0; i < numJobManagers; i++) {
- try {
- runners[i] = new JobManagerRunner(job, configuration,
- rpcService, haServices, jobManagerServices, metricRegistry,
- onCompletion, errorHandler);
- runners[i].start();
- }
- catch (Throwable t) {
- // shut down all the ones so far
- for (int k = 0; k <= i; k++) {
- try {
- if (runners[i] != null) {
- runners[i].shutdown();
- }
- } catch (Throwable ignored) {
- // silent shutdown
- }
- }
-
- throw new JobExecutionException(job.getJobID(), "Could not start the JobManager(s) for the job", t);
- }
- }
-
- return runners;
- }
-
- // ------------------------------------------------------------------------
- // test methods to simulate job master failures
- // ------------------------------------------------------------------------
-
-// public void killJobMaster(int which) {
-// checkArgument(which >= 0 && which < numJobManagers, "no such job master");
-// checkState(!shutdown, "mini cluster is shut down");
-//
-// JobManagerRunner[] runners = this.runners;
-// checkState(runners != null, "mini cluster it not executing a job right now");
-//
-// runners[which].shutdown(new Throwable("kill JobManager"));
-// }
-
- // ------------------------------------------------------------------------
- // utility classes
- // ------------------------------------------------------------------------
-
- /**
- * Simple class that waits for all runners to have reported that they are done.
- * In the case of a high-availability test setup, there may be multiple runners.
- * After that, it marks the mini cluster as ready to receive new jobs.
- */
- private class DetachedFinalizer implements OnCompletionActions, FatalErrorHandler {
-
- private final AtomicInteger numJobManagersToWaitFor;
-
- private DetachedFinalizer(int numJobManagersToWaitFor) {
- this.numJobManagersToWaitFor = new AtomicInteger(numJobManagersToWaitFor);
- }
-
- @Override
- public void jobFinished(JobExecutionResult result) {
- decrementCheckAndCleanup();
- }
-
- @Override
- public void jobFailed(Throwable cause) {
- decrementCheckAndCleanup();
- }
-
- @Override
- public void jobFinishedByOther() {
- decrementCheckAndCleanup();
- }
-
- @Override
- public void onFatalError(Throwable exception) {
- decrementCheckAndCleanup();
- }
-
- private void decrementCheckAndCleanup() {
- if (numJobManagersToWaitFor.decrementAndGet() == 0) {
- MiniClusterJobDispatcher.this.runners = null;
- }
- }
- }
-
- // ------------------------------------------------------------------------
-
- /**
- * This class is used to sync on blocking jobs across multiple runners.
- * Only after all runners reported back that they are finished, the
- * result will be released.
- *
- * That way it is guaranteed that after the blocking job submit call returns,
- * the dispatcher is immediately free to accept another job.
- */
- private static class BlockingJobSync implements OnCompletionActions, FatalErrorHandler {
-
- private final JobID jobId;
-
- private final CountDownLatch jobMastersToWaitFor;
-
- private volatile Throwable jobException;
-
- private volatile Throwable runnerException;
-
- private volatile JobExecutionResult result;
-
- BlockingJobSync(JobID jobId, int numJobMastersToWaitFor) {
- this.jobId = jobId;
- this.jobMastersToWaitFor = new CountDownLatch(numJobMastersToWaitFor);
- }
-
- @Override
- public void jobFinished(JobExecutionResult jobResult) {
- this.result = jobResult;
- jobMastersToWaitFor.countDown();
- }
-
- @Override
- public void jobFailed(Throwable cause) {
- jobException = cause;
- jobMastersToWaitFor.countDown();
- }
-
- @Override
- public void jobFinishedByOther() {
- this.jobMastersToWaitFor.countDown();
- }
-
- @Override
- public void onFatalError(Throwable exception) {
- if (runnerException == null) {
- runnerException = exception;
- }
- }
-
- public JobExecutionResult getResult() throws JobExecutionException, InterruptedException {
- jobMastersToWaitFor.await();
-
- final Throwable jobFailureCause = this.jobException;
- final Throwable runnerException = this.runnerException;
- final JobExecutionResult result = this.result;
-
- // (1) we check if the job terminated with an exception
- // (2) we check whether the job completed successfully
- // (3) we check if we have exceptions from the JobManagers. the job may still have
- // completed successfully in that case, if multiple JobMasters were running
- // and other took over. only if all encounter a fatal error, the job cannot finish
-
- if (jobFailureCause != null) {
- if (jobFailureCause instanceof JobExecutionException) {
- throw (JobExecutionException) jobFailureCause;
- }
- else {
- throw new JobExecutionException(jobId, "The job execution failed", jobFailureCause);
- }
- }
- else if (result != null) {
- return result;
- }
- else if (runnerException != null) {
- throw new JobExecutionException(jobId,
- "The job execution failed because all JobManagers encountered fatal errors", runnerException);
- }
- else {
- throw new IllegalStateException("Bug: Job finished with neither error nor result.");
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/cd069f00/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
new file mode 100644
index 0000000..1ee38e0
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
@@ -0,0 +1,406 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.minicluster;
+
+import akka.actor.ActorSystem;
+
+import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.UnmodifiableConfiguration;
+import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
+import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
+import org.apache.flink.util.ExceptionUtils;
+
+import scala.Option;
+import scala.Tuple2;
+
+import javax.annotation.concurrent.GuardedBy;
+
+import static org.apache.flink.util.ExceptionUtils.firstOrSuppressed;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
+
+public class MiniCluster {
+
+ /** The lock to guard startup / shutdown / manipulation methods */
+ private final Object lock = new Object();
+
+ /** The configuration for this mini cluster */
+ private final MiniClusterConfiguration config;
+
+ @GuardedBy("lock")
+ private MetricRegistry metricRegistry;
+
+ @GuardedBy("lock")
+ private RpcService commonRpcService;
+
+ @GuardedBy("lock")
+ private RpcService[] jobManagerRpcServices;
+
+ @GuardedBy("lock")
+ private RpcService[] taskManagerRpcServices;
+
+ @GuardedBy("lock")
+ private HighAvailabilityServices haServices;
+
+ @GuardedBy("lock")
+ private MiniClusterJobDispatcher jobDispatcher;
+
+ /** Flag marking the mini cluster as started/running */
+ @GuardedBy("lock")
+ private boolean running;
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * Creates a new mini cluster with the default configuration:
+ * <ul>
+ * <li>One JobManager</li>
+ * <li>One TaskManager</li>
+ * <li>One task slot in the TaskManager</li>
+ * <li>All components share the same RPC subsystem (minimizes communication overhead)</li>
+ * </ul>
+ */
+ public MiniCluster() {
+ this(new MiniClusterConfiguration());
+ }
+
+ /**
+ *
+ * @param config The configuration for the mini cluster
+ */
+ public MiniCluster(MiniClusterConfiguration config) {
+ this.config = checkNotNull(config, "config may not be null");
+ }
+
+ /**
+ * Creates a mini cluster based on the given configuration.
+ *
+ * @deprecated Use {@link #MiniCluster(MiniClusterConfiguration)} instead.
+ * @see #MiniCluster(MiniClusterConfiguration)
+ */
+ @Deprecated
+ public MiniCluster(Configuration config) {
+ this(createConfig(config, true));
+ }
+
+ /**
+ * Creates a mini cluster based on the given configuration, starting one or more
+ * RPC services, depending on the given flag.
+ *
+ * @deprecated Use {@link #MiniCluster(MiniClusterConfiguration)} instead.
+ * @see #MiniCluster(MiniClusterConfiguration)
+ */
+ @Deprecated
+ public MiniCluster(Configuration config, boolean singleRpcService) {
+ this(createConfig(config, singleRpcService));
+ }
+
+ // ------------------------------------------------------------------------
+ // life cycle
+ // ------------------------------------------------------------------------
+
+ /**
+ * Checks if the mini cluster was started and is running.
+ */
+ public boolean isRunning() {
+ return running;
+ }
+
+ /**
+ * Starts the mini cluster, based on the configured properties.
+ *
+ * @throws Exception This method passes on any exception that occurs during the startup of
+ * the mini cluster.
+ */
+ public void start() throws Exception {
+ synchronized (lock) {
+ checkState(!running, "FlinkMiniCluster is already running");
+
+ final Configuration configuration = new UnmodifiableConfiguration(config.getConfiguration());
+ final Time rpcTimeout = config.getRpcTimeout();
+ final int numJobManagers = config.getNumJobManagers();
+ final int numTaskManagers = config.getNumTaskManagers();
+ final boolean singleRpc = config.getUseSingleRpcSystem();
+
+ try {
+ metricRegistry = createMetricRegistry(configuration);
+
+ RpcService[] jobManagerRpcServices = new RpcService[numJobManagers];
+ RpcService[] taskManagerRpcServices = new RpcService[numTaskManagers];
+
+ // bring up all the RPC services
+ if (singleRpc) {
+ // one common RPC for all
+ commonRpcService = createRpcService(configuration, rpcTimeout, false, null);
+
+ // set that same RPC service for all JobManagers and TaskManagers
+ for (int i = 0; i < numJobManagers; i++) {
+ jobManagerRpcServices[i] = commonRpcService;
+ }
+ for (int i = 0; i < numTaskManagers; i++) {
+ taskManagerRpcServices[i] = commonRpcService;
+ }
+ }
+ else {
+ // start a new service per component, possibly with custom bind addresses
+ final String jobManagerBindAddress = config.getJobManagerBindAddress();
+ final String taskManagerBindAddress = config.getTaskManagerBindAddress();
+
+ for (int i = 0; i < numJobManagers; i++) {
+ jobManagerRpcServices[i] = createRpcService(
+ configuration, rpcTimeout, true, jobManagerBindAddress);
+ }
+
+ for (int i = 0; i < numTaskManagers; i++) {
+ taskManagerRpcServices[i] = createRpcService(
+ configuration, rpcTimeout, true, taskManagerBindAddress);
+ }
+
+ this.jobManagerRpcServices = jobManagerRpcServices;
+ this.taskManagerRpcServices = taskManagerRpcServices;
+ }
+
+ // create the high-availability services
+ haServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(configuration);
+
+ // bring up the dispatcher that launches JobManagers when jobs submitted
+ jobDispatcher = new MiniClusterJobDispatcher(
+ configuration, haServices, metricRegistry, numJobManagers, jobManagerRpcServices);
+ }
+ catch (Exception e) {
+ // cleanup everything
+ try {
+ shutdownInternally();
+ } catch (Exception ee) {
+ e.addSuppressed(ee);
+ }
+ throw e;
+ }
+
+ // now officially mark this as running
+ running = true;
+ }
+ }
+
+ /**
+ * Shuts down the mini cluster, failing all currently executing jobs.
+ * The mini cluster can be started again by calling the {@link #start()} method again.
+ *
+ * <p>This method shuts down all started services and components,
+ * even if an exception occurs in the process of shutting down some component.
+ *
+ * @throws Exception Thrown, if the shutdown did not complete cleanly.
+ */
+ public void shutdown() throws Exception {
+ synchronized (lock) {
+ if (running) {
+ try {
+ shutdownInternally();
+ } finally {
+ running = false;
+ }
+ }
+ }
+ }
+
+ @GuardedBy("lock")
+ private void shutdownInternally() throws Exception {
+ // this should always be called under the lock
+ assert Thread.holdsLock(lock);
+
+ // collect the first exception, but continue and add all successive
+ // exceptions as suppressed
+ Throwable exception = null;
+
+ // cancel all jobs and shut down the job dispatcher
+ if (jobDispatcher != null) {
+ try {
+ jobDispatcher.shutdown();
+ } catch (Exception e) {
+ exception = firstOrSuppressed(e, exception);
+ }
+ jobDispatcher = null;
+ }
+
+ // shut down high-availability services
+ if (haServices != null) {
+ try {
+ haServices.shutdown();
+ } catch (Exception e) {
+ exception = firstOrSuppressed(e, exception);
+ }
+ haServices = null;
+ }
+
+ // shut down the RpcServices
+ if (commonRpcService != null) {
+ exception = shutDownRpc(commonRpcService, exception);
+ commonRpcService = null;
+ }
+ if (jobManagerRpcServices != null) {
+ for (RpcService service : jobManagerRpcServices) {
+ exception = shutDownRpc(service, exception);
+ }
+ jobManagerRpcServices = null;
+ }
+ if (taskManagerRpcServices != null) {
+ for (RpcService service : taskManagerRpcServices) {
+ exception = shutDownRpc(service, exception);
+ }
+ taskManagerRpcServices = null;
+ }
+
+ // metrics shutdown
+ if (metricRegistry != null) {
+ metricRegistry.shutdown();
+ metricRegistry = null;
+ }
+
+ // if anything went wrong, throw the first error with all the additional suppressed exceptions
+ if (exception != null) {
+ ExceptionUtils.rethrowException(exception, "Error while shutting down mini cluster");
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // running jobs
+ // ------------------------------------------------------------------------
+
+ /**
+ * This method executes a job in detached mode. The method returns immediately after the job
+ * has been added to the
+ *
+ * @param job The Flink job to execute
+ *
+ * @throws JobExecutionException Thrown if anything went amiss during initial job launch,
+ * or if the job terminally failed.
+ */
+ public void runDetached(JobGraph job) throws JobExecutionException {
+ checkNotNull(job, "job is null");
+
+ synchronized (lock) {
+ checkState(running, "mini cluster is not running");
+ jobDispatcher.runDetached(job);
+ }
+ }
+
+ /**
+ * This method runs a job in blocking mode. The method returns only after the job
+ * completed successfully, or after it failed terminally.
+ *
+ * @param job The Flink job to execute
+ * @return The result of the job execution
+ *
+ * @throws JobExecutionException Thrown if anything went amiss during initial job launch,
+ * or if the job terminally failed.
+ */
+ public JobExecutionResult runJobBlocking(JobGraph job) throws JobExecutionException, InterruptedException {
+ checkNotNull(job, "job is null");
+
+ MiniClusterJobDispatcher dispatcher;
+ synchronized (lock) {
+ checkState(running, "mini cluster is not running");
+ dispatcher = this.jobDispatcher;
+ }
+
+ return dispatcher.runJobBlocking(job);
+ }
+
+ // ------------------------------------------------------------------------
+ // factories - can be overridden by subclasses to alter behavior
+ // ------------------------------------------------------------------------
+
+ /**
+ * Factory method to create the metric registry for the mini cluster
+ *
+ * @param config The configuration of the mini cluster
+ */
+ protected MetricRegistry createMetricRegistry(Configuration config) {
+ return new MetricRegistry(MetricRegistryConfiguration.fromConfiguration(config));
+ }
+
+ /**
+ * Factory method to instantiate the RPC service.
+ *
+ * @param config
+ * The configuration of the mini cluster
+ * @param askTimeout
+ * The default RPC timeout for asynchronous "ask" requests.
+ * @param remoteEnabled
+ * True, if the RPC service should be reachable from other (remote) RPC services.
+ * @param bindAddress
+ * The address to bind the RPC service to. Only relevant when "remoteEnabled" is true.
+ *
+ * @return The instantiated RPC service
+ */
+ protected RpcService createRpcService(
+ Configuration config,
+ Time askTimeout,
+ boolean remoteEnabled,
+ String bindAddress) {
+
+ ActorSystem actorSystem;
+ if (remoteEnabled) {
+ Tuple2<String, Object> remoteSettings = new Tuple2<String, Object>(bindAddress, 0);
+ actorSystem = AkkaUtils.createActorSystem(config, Option.apply(remoteSettings));
+ } else {
+ actorSystem = AkkaUtils.createLocalActorSystem(config);
+ }
+
+ return new AkkaRpcService(actorSystem, askTimeout);
+ }
+
+ // ------------------------------------------------------------------------
+ // miscellaneous utilities
+ // ------------------------------------------------------------------------
+
+ private static Throwable shutDownRpc(RpcService rpcService, Throwable priorException) {
+ try {
+ if (rpcService != null) {
+ rpcService.stopService();
+ }
+ return priorException;
+ }
+ catch (Throwable t) {
+ return firstOrSuppressed(t, priorException);
+ }
+ }
+
+ private static MiniClusterConfiguration createConfig(Configuration cfg, boolean singleActorSystem) {
+ MiniClusterConfiguration config = cfg == null ?
+ new MiniClusterConfiguration() :
+ new MiniClusterConfiguration(cfg);
+
+ if (!singleActorSystem) {
+ config.setUseRpcServicePerComponent();
+ }
+
+ return config;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/cd069f00/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java
new file mode 100644
index 0000000..a8d7b10
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.minicluster;
+
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.akka.AkkaUtils;
+import scala.concurrent.duration.FiniteDuration;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+public class MiniClusterConfiguration {
+
+ private final Configuration config;
+
+ private boolean singleRpcService = true;
+
+ private int numJobManagers = 1;
+
+ private int numTaskManagers = 1;
+
+ private String commonBindAddress;
+
+ // ------------------------------------------------------------------------
+ // Construction
+ // ------------------------------------------------------------------------
+
+ public MiniClusterConfiguration() {
+ this.config = new Configuration();
+ }
+
+ public MiniClusterConfiguration(Configuration config) {
+ checkNotNull(config);
+ this.config = new Configuration(config);
+ }
+
+ // ------------------------------------------------------------------------
+ // setters
+ // ------------------------------------------------------------------------
+
+ public void addConfiguration(Configuration config) {
+ checkNotNull(config, "configuration must not be null");
+ this.config.addAll(config);
+ }
+
+ public void setUseSingleRpcService() {
+ this.singleRpcService = true;
+ }
+
+ public void setUseRpcServicePerComponent() {
+ this.singleRpcService = false;
+ }
+
+ public void setNumJobManagers(int numJobManagers) {
+ checkArgument(numJobManagers >= 1, "must have at least one JobManager");
+ this.numJobManagers = numJobManagers;
+ }
+
+ public void setNumTaskManagers(int numTaskManagers) {
+ checkArgument(numTaskManagers >= 1, "must have at least one TaskManager");
+ this.numTaskManagers = numTaskManagers;
+ }
+
+ public void setNumTaskManagerSlots(int numTaskSlots) {
+ checkArgument(numTaskSlots >= 1, "must have at least one task slot per TaskManager");
+ this.config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, numTaskSlots);
+ }
+
+ public void setCommonRpcBindAddress(String bindAddress) {
+ checkNotNull(bindAddress, "bind address must not be null");
+ this.commonBindAddress = bindAddress;
+ }
+
+ // ------------------------------------------------------------------------
+ // getters
+ // ------------------------------------------------------------------------
+
+ public Configuration getConfiguration() {
+ return config;
+ }
+
+ public boolean getUseSingleRpcSystem() {
+ return singleRpcService;
+ }
+
+ public int getNumJobManagers() {
+ return numJobManagers;
+ }
+
+ public int getNumTaskManagers() {
+ return numTaskManagers;
+ }
+
+ public int getNumSlotsPerTaskManager() {
+ return config.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
+ }
+
+ public String getJobManagerBindAddress() {
+ return commonBindAddress != null ?
+ commonBindAddress :
+ config.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost");
+ }
+
+ public String getTaskManagerBindAddress() {
+ return commonBindAddress != null ?
+ commonBindAddress :
+ config.getString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, "localhost");
+ }
+
+ public Time getRpcTimeout() {
+ FiniteDuration duration = AkkaUtils.getTimeout(config);
+ return Time.of(duration.length(), duration.unit());
+ }
+
+ // ------------------------------------------------------------------------
+ // utils
+ // ------------------------------------------------------------------------
+
+ @Override
+ public String toString() {
+ return "MiniClusterConfiguration{" +
+ "singleRpcService=" + singleRpcService +
+ ", numJobManagers=" + numJobManagers +
+ ", numTaskManagers=" + numTaskManagers +
+ ", commonBindAddress='" + commonBindAddress + '\'' +
+ ", config=" + config +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/cd069f00/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterJobDispatcher.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterJobDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterJobDispatcher.java
new file mode 100644
index 0000000..d99eff6
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterJobDispatcher.java
@@ -0,0 +1,418 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.minicluster;
+
+import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobmanager.OnCompletionActions;
+import org.apache.flink.runtime.jobmaster.JobManagerRunner;
+import org.apache.flink.runtime.jobmaster.JobManagerServices;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
+import org.apache.flink.runtime.rpc.RpcService;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
+/**
+ * The dispatcher that runs in the mini cluster, waits for jobs, and starts job masters
+ * upon receiving jobs.
+ */
+public class MiniClusterJobDispatcher {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MiniClusterJobDispatcher.class);
+
+ // ------------------------------------------------------------------------
+
+ /** lock to ensure that this dispatcher executes only one job at a time */
+ private final Object lock = new Object();
+
+ /** the configuration with which the mini cluster was started */
+ private final Configuration configuration;
+
+ /** the RPC services to use by the job managers */
+ private final RpcService[] rpcServices;
+
+ /** services for discovery, leader election, and recovery */
+ private final HighAvailabilityServices haServices;
+
+ /** al the services that the JobManager needs, such as BLOB service, factories, etc */
+ private final JobManagerServices jobManagerServices;
+
+ /** Registry for all metrics in the mini cluster */
+ private final MetricRegistry metricRegistry;
+
+ /** The number of JobManagers to launch (more than one simulates a high-availability setup) */
+ private final int numJobManagers;
+
+ /** The runner for the job and master. non-null if a job is currently running */
+ private volatile JobManagerRunner[] runners;
+
+ /** flag marking the dispatcher as hut down */
+ private volatile boolean shutdown;
+
+
+ /**
+ * Starts a mini cluster job dispatcher.
+ *
+ * <p>The dispatcher kicks off one JobManager per job, a behavior similar to a
+ * non-highly-available setup.
+ *
+ * @param config The configuration of the mini cluster
+ * @param haServices Access to the discovery, leader election, and recovery services
+ *
+ * @throws Exception Thrown, if the services for the JobMaster could not be started.
+ */
+ public MiniClusterJobDispatcher(
+ Configuration config,
+ RpcService rpcService,
+ HighAvailabilityServices haServices,
+ MetricRegistry metricRegistry) throws Exception {
+ this(config, haServices, metricRegistry, 1, new RpcService[] { rpcService });
+ }
+
+ /**
+ * Starts a mini cluster job dispatcher.
+ *
+ * <p>The dispatcher may kick off more than one JobManager per job, thus simulating
+ * a highly-available setup.
+ *
+ * @param config The configuration of the mini cluster
+ * @param haServices Access to the discovery, leader election, and recovery services
+ * @param numJobManagers The number of JobMasters to start for each job.
+ *
+ * @throws Exception Thrown, if the services for the JobMaster could not be started.
+ */
+ public MiniClusterJobDispatcher(
+ Configuration config,
+ HighAvailabilityServices haServices,
+ MetricRegistry metricRegistry,
+ int numJobManagers,
+ RpcService[] rpcServices) throws Exception {
+
+ checkArgument(numJobManagers >= 1);
+ checkArgument(rpcServices.length == numJobManagers);
+
+ this.configuration = checkNotNull(config);
+ this.rpcServices = rpcServices;
+ this.haServices = checkNotNull(haServices);
+ this.metricRegistry = checkNotNull(metricRegistry);
+ this.numJobManagers = numJobManagers;
+
+ LOG.info("Creating JobMaster services");
+ this.jobManagerServices = JobManagerServices.fromConfiguration(config, haServices);
+ }
+
+ // ------------------------------------------------------------------------
+ // life cycle
+ // ------------------------------------------------------------------------
+
+ /**
+ * Shuts down the mini cluster dispatcher. If a job is currently running, that job will be
+ * terminally failed.
+ */
+ public void shutdown() {
+ synchronized (lock) {
+ if (!shutdown) {
+ shutdown = true;
+
+ LOG.info("Shutting down the dispatcher");
+
+ // in this shutdown code we copy the references to the stack first,
+ // to avoid concurrent modification
+
+ JobManagerRunner[] runners = this.runners;
+ if (runners != null) {
+ this.runners = null;
+
+ for (JobManagerRunner runner : runners) {
+ runner.shutdown();
+ }
+ }
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // submitting jobs
+ // ------------------------------------------------------------------------
+
+ /**
+ * This method executes a job in detached mode. The method returns immediately after the job
+ * has been added to the
+ *
+ * @param job The Flink job to execute
+ *
+ * @throws JobExecutionException Thrown if anything went amiss during initial job launch,
+ * or if the job terminally failed.
+ */
+ public void runDetached(JobGraph job) throws JobExecutionException {
+ checkNotNull(job);
+
+ LOG.info("Received job for detached execution {} ({})", job.getName(), job.getJobID());
+
+ synchronized (lock) {
+ checkState(!shutdown, "mini cluster is shut down");
+ checkState(runners == null, "mini cluster can only execute one job at a time");
+
+ DetachedFinalizer finalizer = new DetachedFinalizer(numJobManagers);
+
+ this.runners = startJobRunners(job, finalizer, finalizer);
+ }
+ }
+
+ /**
+ * This method runs a job in blocking mode. The method returns only after the job
+ * completed successfully, or after it failed terminally.
+ *
+ * @param job The Flink job to execute
+ * @return The result of the job execution
+ *
+ * @throws JobExecutionException Thrown if anything went amiss during initial job launch,
+ * or if the job terminally failed.
+ */
+ public JobExecutionResult runJobBlocking(JobGraph job) throws JobExecutionException, InterruptedException {
+ checkNotNull(job);
+
+ LOG.info("Received job for blocking execution {} ({})", job.getName(), job.getJobID());
+ final BlockingJobSync sync = new BlockingJobSync(job.getJobID(), numJobManagers);
+
+ synchronized (lock) {
+ checkState(!shutdown, "mini cluster is shut down");
+ checkState(runners == null, "mini cluster can only execute one job at a time");
+
+ this.runners = startJobRunners(job, sync, sync);
+ }
+
+ try {
+ return sync.getResult();
+ }
+ finally {
+ // always clear the status for the next job
+ runners = null;
+ }
+ }
+
+ private JobManagerRunner[] startJobRunners(
+ JobGraph job,
+ OnCompletionActions onCompletion,
+ FatalErrorHandler errorHandler) throws JobExecutionException {
+
+ LOG.info("Starting {} JobMaster(s) for job {} ({})", numJobManagers, job.getName(), job.getJobID());
+
+ // we first need to mark the job as running in the HA services, so that the
+ // JobManager leader will recognize that it as work to do
+ try {
+ haServices.getRunningJobsRegistry().setJobRunning(job.getJobID());
+ }
+ catch (Throwable t) {
+ throw new JobExecutionException(job.getJobID(),
+ "Could not register the job at the high-availability services", t);
+ }
+
+ // start all JobManagers
+ JobManagerRunner[] runners = new JobManagerRunner[numJobManagers];
+ for (int i = 0; i < numJobManagers; i++) {
+ try {
+ runners[i] = new JobManagerRunner(job, configuration,
+ rpcServices[i], haServices, jobManagerServices, metricRegistry,
+ onCompletion, errorHandler);
+ runners[i].start();
+ }
+ catch (Throwable t) {
+ // shut down all the ones so far
+ for (int k = 0; k <= i; k++) {
+ try {
+ if (runners[i] != null) {
+ runners[i].shutdown();
+ }
+ } catch (Throwable ignored) {
+ // silent shutdown
+ }
+ }
+
+ // un-register the job from the high.availability services
+ try {
+ haServices.getRunningJobsRegistry().setJobFinished(job.getJobID());
+ }
+ catch (Throwable tt) {
+ LOG.warn("Could not properly unregister job from high-availability services", tt);
+ }
+
+ throw new JobExecutionException(job.getJobID(), "Could not start the JobManager(s) for the job", t);
+ }
+ }
+
+ return runners;
+ }
+
+ // ------------------------------------------------------------------------
+ // test methods to simulate job master failures
+ // ------------------------------------------------------------------------
+
+// public void killJobMaster(int which) {
+// checkArgument(which >= 0 && which < numJobManagers, "no such job master");
+// checkState(!shutdown, "mini cluster is shut down");
+//
+// JobManagerRunner[] runners = this.runners;
+// checkState(runners != null, "mini cluster it not executing a job right now");
+//
+// runners[which].shutdown(new Throwable("kill JobManager"));
+// }
+
+ // ------------------------------------------------------------------------
+ // utility classes
+ // ------------------------------------------------------------------------
+
+ /**
+ * Simple class that waits for all runners to have reported that they are done.
+ * In the case of a high-availability test setup, there may be multiple runners.
+ * After that, it marks the mini cluster as ready to receive new jobs.
+ */
+ private class DetachedFinalizer implements OnCompletionActions, FatalErrorHandler {
+
+ private final AtomicInteger numJobManagersToWaitFor;
+
+ private DetachedFinalizer(int numJobManagersToWaitFor) {
+ this.numJobManagersToWaitFor = new AtomicInteger(numJobManagersToWaitFor);
+ }
+
+ @Override
+ public void jobFinished(JobExecutionResult result) {
+ decrementCheckAndCleanup();
+ }
+
+ @Override
+ public void jobFailed(Throwable cause) {
+ decrementCheckAndCleanup();
+ }
+
+ @Override
+ public void jobFinishedByOther() {
+ decrementCheckAndCleanup();
+ }
+
+ @Override
+ public void onFatalError(Throwable exception) {
+ decrementCheckAndCleanup();
+ }
+
+ private void decrementCheckAndCleanup() {
+ if (numJobManagersToWaitFor.decrementAndGet() == 0) {
+ MiniClusterJobDispatcher.this.runners = null;
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * This class is used to sync on blocking jobs across multiple runners.
+ * Only after all runners reported back that they are finished, the
+ * result will be released.
+ *
+ * That way it is guaranteed that after the blocking job submit call returns,
+ * the dispatcher is immediately free to accept another job.
+ */
+ private static class BlockingJobSync implements OnCompletionActions, FatalErrorHandler {
+
+ private final JobID jobId;
+
+ private final CountDownLatch jobMastersToWaitFor;
+
+ private volatile Throwable jobException;
+
+ private volatile Throwable runnerException;
+
+ private volatile JobExecutionResult result;
+
+ BlockingJobSync(JobID jobId, int numJobMastersToWaitFor) {
+ this.jobId = jobId;
+ this.jobMastersToWaitFor = new CountDownLatch(numJobMastersToWaitFor);
+ }
+
+ @Override
+ public void jobFinished(JobExecutionResult jobResult) {
+ this.result = jobResult;
+ jobMastersToWaitFor.countDown();
+ }
+
+ @Override
+ public void jobFailed(Throwable cause) {
+ jobException = cause;
+ jobMastersToWaitFor.countDown();
+ }
+
+ @Override
+ public void jobFinishedByOther() {
+ this.jobMastersToWaitFor.countDown();
+ }
+
+ @Override
+ public void onFatalError(Throwable exception) {
+ if (runnerException == null) {
+ runnerException = exception;
+ }
+ }
+
+ public JobExecutionResult getResult() throws JobExecutionException, InterruptedException {
+ jobMastersToWaitFor.await();
+
+ final Throwable jobFailureCause = this.jobException;
+ final Throwable runnerException = this.runnerException;
+ final JobExecutionResult result = this.result;
+
+ // (1) we check if the job terminated with an exception
+ // (2) we check whether the job completed successfully
+ // (3) we check if we have exceptions from the JobManagers. the job may still have
+ // completed successfully in that case, if multiple JobMasters were running
+ // and other took over. only if all encounter a fatal error, the job cannot finish
+
+ if (jobFailureCause != null) {
+ if (jobFailureCause instanceof JobExecutionException) {
+ throw (JobExecutionException) jobFailureCause;
+ }
+ else {
+ throw new JobExecutionException(jobId, "The job execution failed", jobFailureCause);
+ }
+ }
+ else if (result != null) {
+ return result;
+ }
+ else if (runnerException != null) {
+ throw new JobExecutionException(jobId,
+ "The job execution failed because all JobManagers encountered fatal errors", runnerException);
+ }
+ else {
+ throw new IllegalStateException("Bug: Job finished with neither error nor result.");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/cd069f00/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index 6f6d525..3122804 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -188,7 +188,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
} else {
try {
LeaderRetrievalService jobMasterLeaderRetriever =
- highAvailabilityServices.getJobManagerLeaderRetriever(jobID, jobMasterAddress);
+ highAvailabilityServices.getJobManagerLeaderRetriever(jobID);
jobIdLeaderListener = new JobIdLeaderListener(jobID, jobMasterLeaderRetriever);
} catch (Exception e) {
log.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
http://git-wip-us.apache.org/repos/asf/flink/blob/cd069f00/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
index 9e71349..e7f52e2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
@@ -191,8 +191,7 @@ public class JobLeaderService {
LOG.info("Add job {} for job leader monitoring.", jobId);
final LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(
- jobId,
- defaultTargetAddress);
+ jobId);
JobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = new JobManagerLeaderListener(jobId);
http://git-wip-us.apache.org/repos/asf/flink/blob/cd069f00/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
index 3e88e8c..877812b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/TestingHighAvailabilityServices.java
@@ -91,7 +91,7 @@ public class TestingHighAvailabilityServices implements HighAvailabilityServices
}
@Override
- public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) throws Exception {
+ public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) throws Exception {
LeaderRetrievalService service = this.jobMasterLeaderRetrievers.get(jobID);
if (service != null) {
return service;
http://git-wip-us.apache.org/repos/asf/flink/blob/cd069f00/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
new file mode 100644
index 0000000..dd43337
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.minicluster;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.runtime.testtasks.NoOpInvokable;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+/**
+ * Integration test cases for the {@link MiniCluster}.
+ */
+public class MiniClusterITCase extends TestLogger {
+
+// @Test
+ public void runJobWithSingleRpcService() throws Exception {
+ MiniClusterConfiguration cfg = new MiniClusterConfiguration();
+
+ // should be the default, but set anyways to make sure the test
+ // stays valid when the default changes
+ cfg.setUseSingleRpcService();
+
+ MiniCluster miniCluster = new MiniCluster(cfg);
+ executeJob(miniCluster);
+ }
+
+// @Test
+ public void runJobWithMultipleRpcServices() throws Exception {
+ MiniClusterConfiguration cfg = new MiniClusterConfiguration();
+ cfg.setUseRpcServicePerComponent();
+
+ MiniCluster miniCluster = new MiniCluster(cfg);
+ executeJob(miniCluster);
+ }
+
+// @Test
+ public void runJobWithMultipleJobManagers() throws Exception {
+ MiniClusterConfiguration cfg = new MiniClusterConfiguration();
+ cfg.setNumJobManagers(3);
+
+ MiniCluster miniCluster = new MiniCluster(cfg);
+ executeJob(miniCluster);
+ }
+
+ private static void executeJob(MiniCluster miniCluster) throws Exception {
+ miniCluster.start();
+
+ JobGraph job = getSimpleJob();
+ miniCluster.runJobBlocking(job);
+ }
+
+ private static JobGraph getSimpleJob() {
+ JobVertex task = new JobVertex("Test task");
+ task.setParallelism(1);
+ task.setMaxParallelism(1);
+ task.setInvokableClass(NoOpInvokable.class);
+
+ return new JobGraph(new JobID(), "Test Job", task);
+ }
+}
[48/50] [abbrv] flink git commit: [FLINK-4851] [rm] Introduce
FatalErrorHandler and MetricRegistry to RM
Posted by tr...@apache.org.
[FLINK-4851] [rm] Introduce FatalErrorHandler and MetricRegistry to RM
This PR introduces a FatalErrorHandler and the MetricRegistry to the RM. The FatalErrorHandler is used to handle fatal errors. Additionally, the PR adds the MetricRegistry to the RM which can be used
to register metrics.
Apart from these changes the PR restructures the code of the RM a little bit and fixes some
blocking operations.
The PR also moves the TestingFatalErrorHandler into the util package of flink-runtime test. That
it is usable across multiple tests.
Introduce ResourceManagerRunner to handle errors in the ResourceManager
This closes #2655.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/7fab3edf
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/7fab3edf
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/7fab3edf
Branch: refs/heads/flip-6
Commit: 7fab3edf78fad2fc580942e3f5fe69b1f8ded426
Parents: 0262e27
Author: Till Rohrmann <tr...@apache.org>
Authored: Mon Oct 17 16:03:02 2016 +0200
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:35 2016 +0100
----------------------------------------------------------------------
.../flink/runtime/minicluster/MiniCluster.java | 35 +-
.../resourcemanager/ResourceManager.java | 463 +++++++++++--------
.../ResourceManagerConfiguration.java | 86 ++++
.../StandaloneResourceManager.java | 25 +-
.../exceptions/ConfigurationException.java | 38 ++
.../exceptions/ResourceManagerException.java | 40 ++
.../exceptions/ResourceManagerRunner.java | 102 ++++
.../registration/WorkerRegistration.java | 5 +-
.../resourcemanager/ResourceManagerHATest.java | 45 +-
.../ResourceManagerJobMasterTest.java | 59 ++-
.../ResourceManagerTaskExecutorTest.java | 87 ++--
.../resourcemanager/TestingResourceManager.java | 53 ---
.../TestingSlotManagerFactory.java | 30 ++
.../slotmanager/SlotProtocolTest.java | 47 +-
.../runtime/taskexecutor/TaskExecutorTest.java | 60 +--
.../runtime/util/TestingFatalErrorHandler.java | 83 ++++
16 files changed, 863 insertions(+), 395 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
index d63f9a7..b005330 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java
@@ -32,10 +32,7 @@ import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
-import org.apache.flink.runtime.resourcemanager.ResourceManager;
-import org.apache.flink.runtime.resourcemanager.StandaloneResourceManager;
-import org.apache.flink.runtime.resourcemanager.slotmanager.DefaultSlotManager;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerRunner;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.rpc.akka.AkkaRpcService;
import org.apache.flink.runtime.taskexecutor.TaskManagerRunner;
@@ -52,7 +49,6 @@ import static org.apache.flink.util.ExceptionUtils.firstOrSuppressed;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.flink.util.Preconditions.checkState;
-
public class MiniCluster {
private static final Logger LOG = LoggerFactory.getLogger(MiniCluster.class);
@@ -82,7 +78,7 @@ public class MiniCluster {
private HighAvailabilityServices haServices;
@GuardedBy("lock")
- private ResourceManager<?>[] resourceManagers;
+ private ResourceManagerRunner[] resourceManagerRunners;
@GuardedBy("lock")
private TaskManagerRunner[] taskManagerRunners;
@@ -231,7 +227,7 @@ public class MiniCluster {
// bring up the ResourceManager(s)
LOG.info("Starting {} ResourceManger(s)", numResourceManagers);
- resourceManagers = startResourceManagers(
+ resourceManagerRunners = startResourceManagers(
configuration, haServices, metricRegistry, numResourceManagers, resourceManagerRpcServices);
// bring up the TaskManager(s) for the mini cluster
@@ -303,8 +299,8 @@ public class MiniCluster {
jobDispatcher = null;
}
- if (resourceManagers != null) {
- for (ResourceManager<?> rm : resourceManagers) {
+ if (resourceManagerRunners != null) {
+ for (ResourceManagerRunner rm : resourceManagerRunners) {
if (rm != null) {
try {
rm.shutDown();
@@ -313,7 +309,7 @@ public class MiniCluster {
}
}
}
- resourceManagers = null;
+ resourceManagerRunners = null;
}
// shut down the RpcServices
@@ -435,26 +431,27 @@ public class MiniCluster {
return new AkkaRpcService(actorSystem, askTimeout);
}
- protected ResourceManager<?>[] startResourceManagers(
+ protected ResourceManagerRunner[] startResourceManagers(
Configuration configuration,
HighAvailabilityServices haServices,
MetricRegistry metricRegistry,
int numResourceManagers,
RpcService[] resourceManagerRpcServices) throws Exception {
- final StandaloneResourceManager[] resourceManagers = new StandaloneResourceManager[numResourceManagers];
- final SlotManagerFactory slotManagerFactory = new DefaultSlotManager.Factory();
+ final ResourceManagerRunner[] resourceManagerRunners = new ResourceManagerRunner[numResourceManagers];
for (int i = 0; i < numResourceManagers; i++) {
- resourceManagers[i] = new StandaloneResourceManager(
- resourceManagerRpcServices[i],
- haServices,
- slotManagerFactory);
- resourceManagers[i].start();
+ resourceManagerRunners[i] = new ResourceManagerRunner(
+ configuration,
+ resourceManagerRpcServices[i],
+ haServices,
+ metricRegistry);
+
+ resourceManagerRunners[i].start();
}
- return resourceManagers;
+ return resourceManagerRunners;
}
protected TaskManagerRunner[] startTaskManagers(
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
index f1a5073..4161972 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java
@@ -38,12 +38,15 @@ import org.apache.flink.runtime.leaderelection.LeaderContender;
import org.apache.flink.runtime.leaderelection.LeaderElectionService;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener;
import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestRejected;
import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestReply;
import org.apache.flink.runtime.resourcemanager.registration.JobMasterRegistration;
import org.apache.flink.runtime.resourcemanager.registration.WorkerRegistration;
import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcEndpoint;
import org.apache.flink.runtime.rpc.RpcMethod;
import org.apache.flink.runtime.rpc.RpcService;
@@ -54,6 +57,7 @@ import org.apache.flink.runtime.registration.RegistrationResponse;
import org.apache.flink.runtime.taskexecutor.SlotReport;
import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
import org.apache.flink.runtime.taskexecutor.TaskExecutorRegistrationSuccess;
+import org.apache.flink.util.ExceptionUtils;
import java.io.Serializable;
import java.util.HashMap;
@@ -61,6 +65,8 @@ import java.util.Iterator;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executor;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -79,8 +85,8 @@ public abstract class ResourceManager<WorkerType extends Serializable>
extends RpcEndpoint<ResourceManagerGateway>
implements LeaderContender {
- /** The exit code with which the process is stopped in case of a fatal error. */
- protected static final int EXIT_CODE_FATAL_ERROR = -13;
+ /** Configuration of the resource manager */
+ private final ResourceManagerConfiguration resourceManagerConfiguration;
/** All currently registered JobMasterGateways scoped by JobID. */
private final Map<JobID, JobMasterRegistration> jobMasters;
@@ -97,6 +103,12 @@ public abstract class ResourceManager<WorkerType extends Serializable>
/** The factory to construct the SlotManager. */
private final SlotManagerFactory slotManagerFactory;
+ /** Registry to use for metrics */
+ private final MetricRegistry metricRegistry;
+
+ /** Fatal error handler */
+ private final FatalErrorHandler fatalErrorHandler;
+
/** The SlotManager created by the slotManagerFactory when the ResourceManager is started. */
private SlotManager slotManager;
@@ -107,64 +119,89 @@ public abstract class ResourceManager<WorkerType extends Serializable>
private volatile UUID leaderSessionID;
/** All registered listeners for status updates of the ResourceManager. */
- private Map<String, InfoMessageListenerRpcGateway> infoMessageListeners;
-
- /** Default timeout for messages */
- private final Time timeout = Time.seconds(5);
+ private ConcurrentMap<String, InfoMessageListenerRpcGateway> infoMessageListeners;
public ResourceManager(
RpcService rpcService,
+ ResourceManagerConfiguration resourceManagerConfiguration,
HighAvailabilityServices highAvailabilityServices,
- SlotManagerFactory slotManagerFactory) {
+ SlotManagerFactory slotManagerFactory,
+ MetricRegistry metricRegistry,
+ FatalErrorHandler fatalErrorHandler) {
+
super(rpcService);
+
+ this.resourceManagerConfiguration = checkNotNull(resourceManagerConfiguration);
this.highAvailabilityServices = checkNotNull(highAvailabilityServices);
this.slotManagerFactory = checkNotNull(slotManagerFactory);
- this.jobMasters = new HashMap<>();
- this.leaderListeners = new HashMap<>();
- this.taskExecutors = new HashMap<>();
- this.leaderSessionID = new UUID(0, 0);
- infoMessageListeners = new HashMap<>();
+ this.metricRegistry = checkNotNull(metricRegistry);
+ this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
+
+ this.jobMasters = new HashMap<>(4);
+ this.leaderListeners = new HashMap<>(4);
+ this.taskExecutors = new HashMap<>(8);
+ this.leaderSessionID = null;
+ infoMessageListeners = new ConcurrentHashMap<>(8);
}
+ // ------------------------------------------------------------------------
+ // RPC lifecycle methods
+ // ------------------------------------------------------------------------
+
@Override
- public void start() {
+ public void start() throws Exception {
// start a leader
+ super.start();
+
try {
- super.start();
// SlotManager should start first
slotManager = slotManagerFactory.create(createResourceManagerServices());
+ } catch (Exception e) {
+ throw new ResourceManagerException("Could not create the slot manager.", e);
+ }
+
+ try {
leaderElectionService = highAvailabilityServices.getResourceManagerLeaderElectionService();
leaderElectionService.start(this);
+ } catch (Exception e) {
+ throw new ResourceManagerException("Could not start the leader election service.", e);
+ }
+
+ try {
// framework specific initialization
initialize();
- } catch (Throwable e) {
- log.error("A fatal error happened when starting the ResourceManager", e);
- throw new RuntimeException("A fatal error happened when starting the ResourceManager", e);
+ } catch (Exception e) {
+ throw new ResourceManagerException("Could not initialize the resource manager.", e);
}
}
@Override
- public void shutDown() {
+ public void shutDown() throws Exception {
+ Exception exception = null;
+
try {
leaderElectionService.stop();
- clearState();
+ } catch (Exception e) {
+ exception = ExceptionUtils.firstOrSuppressed(e, exception);
+ }
+
+ clearState();
+
+ try {
super.shutDown();
- } catch (Throwable e) {
- log.error("A fatal error happened when shutdown the ResourceManager", e);
- throw new RuntimeException("A fatal error happened when shutdown the ResourceManager", e);
+ } catch (Exception e) {
+ exception = ExceptionUtils.firstOrSuppressed(e, exception);
}
- }
- /**
- * Gets the leader session id of current resourceManager.
- *
- * @return return the leaderSessionId of current resourceManager, this returns null until the current resourceManager is granted leadership.
- */
- @VisibleForTesting
- UUID getLeaderSessionID() {
- return this.leaderSessionID;
+ if (exception != null) {
+ ExceptionUtils.rethrowException(exception, "Error while shutting the ResourceManager down.");
+ }
}
+ // ------------------------------------------------------------------------
+ // RPC methods
+ // ------------------------------------------------------------------------
+
/**
* Register a {@link JobMaster} at the resource manager.
*
@@ -191,11 +228,12 @@ public abstract class ResourceManager<WorkerType extends Serializable>
highAvailabilityServices.getJobManagerLeaderRetriever(jobID);
jobIdLeaderListener = new JobIdLeaderListener(jobID, jobMasterLeaderRetriever);
} catch (Exception e) {
- log.warn("Failed to start JobMasterLeaderRetriever for JobID {}", jobID);
- FlinkCompletableFuture<RegistrationResponse> responseFuture = new FlinkCompletableFuture<>();
- responseFuture.complete(new RegistrationResponse.Decline("Failed to retrieve JobMasterLeaderRetriever"));
- return responseFuture;
+ log.warn("Failed to start JobMasterLeaderRetriever for job id {}", jobID, e);
+
+ return FlinkCompletableFuture.<RegistrationResponse>completed(
+ new RegistrationResponse.Decline("Failed to retrieve JobMasterLeaderRetriever"));
}
+
leaderListeners.put(jobID, jobIdLeaderListener);
}
@@ -211,6 +249,8 @@ public abstract class ResourceManager<WorkerType extends Serializable>
throw new Exception("Invalid leader session id");
}
+ final Time timeout = resourceManagerConfiguration.getTimeout();
+
if (!jobIdLeaderListener.getLeaderID().get(timeout.getSize(), timeout.getUnit())
.equals(jobMasterLeaderId)) {
throw new Exception("Leader Id did not match");
@@ -224,10 +264,9 @@ public abstract class ResourceManager<WorkerType extends Serializable>
@Override
public RegistrationResponse apply(JobMasterGateway jobMasterGateway, Throwable throwable) {
- if (throwable != null) {
- return new RegistrationResponse.Decline(throwable.getMessage());
- }
-
+ if (throwable != null) {
+ return new RegistrationResponse.Decline(throwable.getMessage());
+ } else {
if (!leaderSessionID.equals(resourceManagerLeaderId)) {
log.warn("Discard registration from JobMaster {} at ({}) because the expected leader session ID {}" +
" did not equal the received leader session ID {}",
@@ -252,10 +291,12 @@ public abstract class ResourceManager<WorkerType extends Serializable>
if (existingRegistration != null) {
log.info("Replacing JobMaster registration for newly registered JobMaster with JobID {}.", jobID);
}
- return new JobMasterRegistrationSuccess(5000, resourceManagerLeaderId);
-
+ return new JobMasterRegistrationSuccess(
+ resourceManagerConfiguration.getHeartbeatInterval().toMilliseconds(),
+ resourceManagerLeaderId);
}
- }, getMainThreadExecutor());
+ }
+ }, getMainThreadExecutor());
}
/**
@@ -274,38 +315,44 @@ public abstract class ResourceManager<WorkerType extends Serializable>
final ResourceID resourceID,
final SlotReport slotReport) {
- return getRpcService().execute(new Callable<TaskExecutorGateway>() {
- @Override
- public TaskExecutorGateway call() throws Exception {
- if (!leaderSessionID.equals(resourceManagerLeaderId)) {
- log.warn("Discard registration from TaskExecutor {} at ({}) because the expected leader session ID {} did " +
- "not equal the received leader session ID {}",
- resourceID, taskExecutorAddress, leaderSessionID, resourceManagerLeaderId);
- throw new Exception("Invalid leader session id");
- }
- return getRpcService().connect(taskExecutorAddress, TaskExecutorGateway.class)
- .get(timeout.toMilliseconds(), timeout.getUnit());
- }
- }).handleAsync(new BiFunction<TaskExecutorGateway, Throwable, RegistrationResponse>() {
- @Override
- public RegistrationResponse apply(TaskExecutorGateway taskExecutorGateway, Throwable throwable) {
- if (throwable != null) {
- return new RegistrationResponse.Decline(throwable.getMessage());
- } else {
- WorkerRegistration oldRegistration = taskExecutors.remove(resourceID);
- if (oldRegistration != null) {
- // TODO :: suggest old taskExecutor to stop itself
- log.info("Replacing old instance of worker for ResourceID {}", resourceID);
+ if (leaderSessionID.equals(resourceManagerLeaderId)) {
+ Future<TaskExecutorGateway> taskExecutorGatewayFuture = getRpcService().connect(taskExecutorAddress, TaskExecutorGateway.class);
+
+ return taskExecutorGatewayFuture.handleAsync(new BiFunction<TaskExecutorGateway, Throwable, RegistrationResponse>() {
+ @Override
+ public RegistrationResponse apply(TaskExecutorGateway taskExecutorGateway, Throwable throwable) {
+ if (throwable != null) {
+ return new RegistrationResponse.Decline(throwable.getMessage());
+ } else {
+ WorkerRegistration<WorkerType> oldRegistration = taskExecutors.remove(resourceID);
+ if (oldRegistration != null) {
+ // TODO :: suggest old taskExecutor to stop itself
+ log.info("Replacing old instance of worker for ResourceID {}", resourceID);
+ }
+
+ WorkerType newWorker = workerStarted(resourceID);
+ WorkerRegistration<WorkerType> registration =
+ new WorkerRegistration<>(taskExecutorGateway, newWorker);
+
+ taskExecutors.put(resourceID, registration);
+ slotManager.registerTaskExecutor(resourceID, registration, slotReport);
+
+ return new TaskExecutorRegistrationSuccess(
+ registration.getInstanceID(),
+ resourceManagerConfiguration.getHeartbeatInterval().toMilliseconds());
}
- WorkerType newWorker = workerStarted(resourceID);
- WorkerRegistration<WorkerType> registration =
- new WorkerRegistration<>(taskExecutorGateway, newWorker);
- taskExecutors.put(resourceID, registration);
- slotManager.registerTaskExecutor(resourceID, registration, slotReport);
- return new TaskExecutorRegistrationSuccess(registration.getInstanceID(), 5000);
}
- }
- }, getMainThreadExecutor());
+ }, getMainThreadExecutor());
+ } else {
+ log.warn("Discard registration from TaskExecutor {} at ({}) because the expected leader session ID {} did " +
+ "not equal the received leader session ID {}",
+ resourceID, taskExecutorAddress, leaderSessionID, resourceManagerLeaderId);
+
+ return FlinkCompletableFuture.<RegistrationResponse>completed(
+ new RegistrationResponse.Decline("Discard registration because the leader id " +
+ resourceManagerLeaderId + " does not match the expected leader id " +
+ leaderSessionID + '.'));
+ }
}
/**
@@ -337,33 +384,91 @@ public abstract class ResourceManager<WorkerType extends Serializable>
* Notification from a TaskExecutor that a slot has become available
* @param resourceManagerLeaderId TaskExecutor's resource manager leader id
* @param instanceID TaskExecutor's instance id
- * @param slotID The slot id of the available slot
+ * @param slotId The slot id of the available slot
* @return SlotAvailableReply
*/
@RpcMethod
public void notifySlotAvailable(
final UUID resourceManagerLeaderId,
final InstanceID instanceID,
- final SlotID slotID) {
+ final SlotID slotId) {
if (resourceManagerLeaderId.equals(leaderSessionID)) {
- final ResourceID resourceId = slotID.getResourceID();
+ final ResourceID resourceId = slotId.getResourceID();
WorkerRegistration<WorkerType> registration = taskExecutors.get(resourceId);
if (registration != null) {
- InstanceID registrationInstanceID = registration.getInstanceID();
- if (registrationInstanceID.equals(instanceID)) {
- runAsync(new Runnable() {
- @Override
- public void run() {
- slotManager.notifySlotAvailable(resourceId, slotID);
- }
- });
+ InstanceID registrationId = registration.getInstanceID();
+
+ if (registrationId.equals(instanceID)) {
+ slotManager.notifySlotAvailable(resourceId, slotId);
+ } else {
+ log.debug("Invalid registration id for slot available message. This indicates an" +
+ " outdated request.");
}
+ } else {
+ log.debug("Could not find registration for resource id {}. Discarding the slot available" +
+ "message {}.", resourceId, slotId);
}
+ } else {
+ log.debug("Discarding notify slot available message for slot {}, because the " +
+ "leader id {} did not match the expected leader id {}.", slotId,
+ resourceManagerLeaderId, leaderSessionID);
+ }
+ }
+
+ /**
+ * Registers an info message listener
+ *
+ * @param address address of infoMessage listener to register to this resource manager
+ */
+ @RpcMethod
+ public void registerInfoMessageListener(final String address) {
+ if(infoMessageListeners.containsKey(address)) {
+ log.warn("Receive a duplicate registration from info message listener on ({})", address);
+ } else {
+ Future<InfoMessageListenerRpcGateway> infoMessageListenerRpcGatewayFuture = getRpcService().connect(address, InfoMessageListenerRpcGateway.class);
+
+ infoMessageListenerRpcGatewayFuture.thenAcceptAsync(new AcceptFunction<InfoMessageListenerRpcGateway>() {
+ @Override
+ public void accept(InfoMessageListenerRpcGateway gateway) {
+ log.info("Receive a registration from info message listener on ({})", address);
+ infoMessageListeners.put(address, gateway);
+ }
+ }, getMainThreadExecutor());
+
+ infoMessageListenerRpcGatewayFuture.exceptionallyAsync(new ApplyFunction<Throwable, Void>() {
+ @Override
+ public Void apply(Throwable failure) {
+ log.warn("Receive a registration from unreachable info message listener on ({})", address);
+ return null;
+ }
+ }, getRpcService().getExecutor());
}
}
+ /**
+ * Unregisters an info message listener
+ *
+ * @param address of the info message listener to unregister from this resource manager
+ *
+ */
+ @RpcMethod
+ public void unRegisterInfoMessageListener(final String address) {
+ infoMessageListeners.remove(address);
+ }
+
+ /**
+ * Cleanup application and shut down cluster
+ *
+ * @param finalStatus
+ * @param optionalDiagnostics
+ */
+ @RpcMethod
+ public void shutDownCluster(final ApplicationStatus finalStatus, final String optionalDiagnostics) {
+ log.info("shut down cluster because application is in {}, diagnostics {}", finalStatus, optionalDiagnostics);
+ shutDownApplication(finalStatus, optionalDiagnostics);
+ }
// ------------------------------------------------------------------------
// Leader Contender
@@ -372,23 +477,35 @@ public abstract class ResourceManager<WorkerType extends Serializable>
/**
* Callback method when current resourceManager is granted leadership
*
- * @param leaderSessionID unique leadershipID
+ * @param newLeaderSessionID unique leadershipID
*/
@Override
- public void grantLeadership(final UUID leaderSessionID) {
+ public void grantLeadership(final UUID newLeaderSessionID) {
runAsync(new Runnable() {
@Override
public void run() {
- log.info("ResourceManager {} was granted leadership with leader session ID {}", getAddress(), leaderSessionID);
- // confirming the leader session ID might be blocking,
- leaderElectionService.confirmLeaderSessionID(leaderSessionID);
- ResourceManager.this.leaderSessionID = leaderSessionID;
+ log.info("ResourceManager {} was granted leadership with leader session ID {}", getAddress(), newLeaderSessionID);
+
+ // clear the state if we've been the leader before
+ if (leaderSessionID != null) {
+ clearState();
+ }
+
+ leaderSessionID = newLeaderSessionID;
+
+ getRpcService().execute(new Runnable() {
+ @Override
+ public void run() {
+ // confirming the leader session ID might be blocking,
+ leaderElectionService.confirmLeaderSessionID(newLeaderSessionID);
+ }
+ });
}
});
}
/**
- * Callback method when current resourceManager lose leadership.
+ * Callback method when current resourceManager loses leadership.
*/
@Override
public void revokeLeadership() {
@@ -396,7 +513,10 @@ public abstract class ResourceManager<WorkerType extends Serializable>
@Override
public void run() {
log.info("ResourceManager {} was revoked leadership.", getAddress());
+
clearState();
+
+ leaderSessionID = null;
}
});
}
@@ -408,106 +528,98 @@ public abstract class ResourceManager<WorkerType extends Serializable>
*/
@Override
public void handleError(final Exception exception) {
- log.error("ResourceManager received an error from the LeaderElectionService.", exception);
- // terminate ResourceManager in case of an error
- shutDown();
+ onFatalErrorAsync(new ResourceManagerException("Received an error from the LeaderElectionService.", exception));
}
/**
- * Registers an infoMessage listener
+ * This method should be called by the framework once it detects that a currently registered
+ * task executor has failed.
*
- * @param infoMessageListenerAddress address of infoMessage listener to register to this resource manager
+ * @param resourceID Id of the worker that has failed.
+ * @param message An informational message that explains why the worker failed.
*/
- @RpcMethod
- public void registerInfoMessageListener(final String infoMessageListenerAddress) {
- if(infoMessageListeners.containsKey(infoMessageListenerAddress)) {
- log.warn("Receive a duplicate registration from info message listener on ({})", infoMessageListenerAddress);
- } else {
- Future<InfoMessageListenerRpcGateway> infoMessageListenerRpcGatewayFuture = getRpcService().connect(infoMessageListenerAddress, InfoMessageListenerRpcGateway.class);
-
- infoMessageListenerRpcGatewayFuture.thenAcceptAsync(new AcceptFunction<InfoMessageListenerRpcGateway>() {
- @Override
- public void accept(InfoMessageListenerRpcGateway gateway) {
- log.info("Receive a registration from info message listener on ({})", infoMessageListenerAddress);
- infoMessageListeners.put(infoMessageListenerAddress, gateway);
- }
- }, getMainThreadExecutor());
+ public void notifyWorkerFailed(final ResourceID resourceID, final String message) {
+ runAsync(new Runnable() {
+ @Override
+ public void run() {
+ WorkerRegistration<WorkerType> workerRegistration = taskExecutors.remove(resourceID);
- infoMessageListenerRpcGatewayFuture.exceptionallyAsync(new ApplyFunction<Throwable, Void>() {
- @Override
- public Void apply(Throwable failure) {
- log.warn("Receive a registration from unreachable info message listener on ({})", infoMessageListenerAddress);
- return null;
+ if (workerRegistration != null) {
+ log.info("Task manager {} failed because {}.", resourceID, message);
+ // TODO :: suggest failed task executor to stop itself
+ slotManager.notifyTaskManagerFailure(resourceID);
+ } else {
+ log.debug("Could not find a registered task manager with the process id {}.", resourceID);
}
- }, getMainThreadExecutor());
- }
- }
-
- /**
- * Unregisters an infoMessage listener
- *
- * @param infoMessageListenerAddress address of infoMessage listener to unregister from this resource manager
- *
- */
- @RpcMethod
- public void unRegisterInfoMessageListener(final String infoMessageListenerAddress) {
- infoMessageListeners.remove(infoMessageListenerAddress);
+ }
+ });
}
- /**
- * Cleanup application and shut down cluster
- *
- * @param finalStatus
- * @param optionalDiagnostics
- */
- @RpcMethod
- public void shutDownCluster(final ApplicationStatus finalStatus, final String optionalDiagnostics) {
- log.info("shut down cluster because application is in {}, diagnostics {}", finalStatus, optionalDiagnostics);
- shutDownApplication(finalStatus, optionalDiagnostics);
- }
+ // ------------------------------------------------------------------------
+ // Error Handling
+ // ------------------------------------------------------------------------
/**
- * This method should be called by the framework once it detects that a currently registered task executor has failed.
+ * Notifies the ResourceManager that a fatal error has occurred and it cannot proceed.
+ * This method should be used when asynchronous threads want to notify the
+ * ResourceManager of a fatal error.
*
- * @param resourceID Id of the worker that has failed.
- * @param message An informational message that explains why the worker failed.
+ * @param t The exception describing the fatal error
*/
- public void notifyWorkerFailed(final ResourceID resourceID, String message) {
+ void onFatalErrorAsync(final Throwable t) {
runAsync(new Runnable() {
@Override
public void run() {
- WorkerType worker = taskExecutors.remove(resourceID).getWorker();
- if (worker != null) {
- // TODO :: suggest failed task executor to stop itself
- slotManager.notifyTaskManagerFailure(resourceID);
- }
+ onFatalError(t);
}
});
}
/**
- * Gets the number of currently started TaskManagers.
+ * Notifies the ResourceManager that a fatal error has occurred and it cannot proceed.
+ * This method must only be called from within the ResourceManager's main thread.
*
- * @return The number of currently started TaskManagers.
+ * @param t The exception describing the fatal error
*/
- public int getNumberOfStartedTaskManagers() {
- return taskExecutors.size();
+ void onFatalError(Throwable t) {
+ log.error("Fatal error occurred.", t);
+ fatalErrorHandler.onFatalError(t);
}
+ // ------------------------------------------------------------------------
+ // Testing methods
+ // ------------------------------------------------------------------------
+
/**
- * Notifies the resource manager of a fatal error.
+ * Gets the leader session id of current resourceManager.
*
- * <p><b>IMPORTANT:</b> This should not cleanly shut down this master, but exit it in
- * such a way that a high-availability setting would restart this or fail over
- * to another master.
+ * @return return the leaderSessionId of current resourceManager, this returns null until the current resourceManager is granted leadership.
*/
- public void onFatalError(final String message, final Throwable error) {
- runAsync(new Runnable() {
- @Override
- public void run() {
- fatalError(message, error);
+ @VisibleForTesting
+ UUID getLeaderSessionID() {
+ return leaderSessionID;
+ }
+
+ // ------------------------------------------------------------------------
+ // Internal methods
+ // ------------------------------------------------------------------------
+
+ private void clearState() {
+ jobMasters.clear();
+ taskExecutors.clear();
+ slotManager.clearState();
+ Iterator<JobIdLeaderListener> leaderListenerIterator =
+ leaderListeners.values().iterator();
+ while (leaderListenerIterator.hasNext()) {
+ JobIdLeaderListener listener = leaderListenerIterator.next();
+ try {
+ listener.stopService();
+ } catch (Exception e) {
+ onFatalError(e);
}
- });
+ leaderListenerIterator.remove();
+ }
+ leaderSessionID = new UUID(0, 0);
}
// ------------------------------------------------------------------------
@@ -522,15 +634,6 @@ public abstract class ResourceManager<WorkerType extends Serializable>
protected abstract void initialize() throws Exception;
/**
- * Notifies the resource master of a fatal error.
- *
- * <p><b>IMPORTANT:</b> This should not cleanly shut down this master, but exit it in
- * such a way that a high-availability setting would restart this or fail over
- * to another master.
- */
- protected abstract void fatalError(String message, Throwable error);
-
- /**
* The framework specific code for shutting down the application. This should report the
* application's final status and shut down the resource manager cleanly.
*
@@ -560,7 +663,7 @@ public abstract class ResourceManager<WorkerType extends Serializable>
// ------------------------------------------------------------------------
public void sendInfoMessage(final String message) {
- runAsync(new Runnable() {
+ getRpcService().execute(new Runnable() {
@Override
public void run() {
InfoMessage infoMessage = new InfoMessage(message);
@@ -675,23 +778,5 @@ public abstract class ResourceManager<WorkerType extends Serializable>
ResourceManager.this.handleError(exception);
}
}
-
- private void clearState() {
- jobMasters.clear();
- taskExecutors.clear();
- slotManager.clearState();
- Iterator<JobIdLeaderListener> leaderListenerIterator =
- leaderListeners.values().iterator();
- while (leaderListenerIterator.hasNext()) {
- JobIdLeaderListener listener = leaderListenerIterator.next();
- try {
- listener.stopService();
- } catch (Exception e) {
- handleError(e);
- }
- leaderListenerIterator.remove();
- }
- leaderSessionID = new UUID(0, 0);
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerConfiguration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerConfiguration.java
new file mode 100644
index 0000000..920f1fc
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerConfiguration.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager;
+
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.resourcemanager.exceptions.ConfigurationException;
+import org.apache.flink.util.Preconditions;
+import scala.concurrent.duration.Duration;
+
+/**
+ * Resource manager configuration
+ */
+public class ResourceManagerConfiguration {
+
+ private final Time timeout;
+ private final Time heartbeatInterval;
+
+ public ResourceManagerConfiguration(Time timeout, Time heartbeatInterval) {
+ this.timeout = Preconditions.checkNotNull(timeout);
+ this.heartbeatInterval = Preconditions.checkNotNull(heartbeatInterval);
+ }
+
+ public Time getTimeout() {
+ return timeout;
+ }
+
+ public Time getHeartbeatInterval() {
+ return heartbeatInterval;
+ }
+
+ // --------------------------------------------------------------------------
+ // Static factory methods
+ // --------------------------------------------------------------------------
+
+ public static ResourceManagerConfiguration fromConfiguration(Configuration configuration) throws ConfigurationException {
+ ConfigOption<String> timeoutOption = ConfigOptions
+ .key(ConfigConstants.AKKA_ASK_TIMEOUT)
+ .defaultValue(ConfigConstants.DEFAULT_AKKA_ASK_TIMEOUT);
+
+ final String strTimeout = configuration.getString(timeoutOption);
+ final Time timeout;
+
+ try {
+ timeout = Time.milliseconds(Duration.apply(strTimeout).toMillis());
+ } catch (NumberFormatException e) {
+ throw new ConfigurationException("Could not parse the resource manager's timeout " +
+ "value " + timeoutOption + '.', e);
+ }
+
+ ConfigOption<String> heartbeatIntervalOption = ConfigOptions
+ .key(ConfigConstants.AKKA_WATCH_HEARTBEAT_INTERVAL)
+ .defaultValue(timeout.toString());
+
+ final String strHeartbeatInterval = configuration.getString(heartbeatIntervalOption);
+ final Time heartbeatInterval;
+
+ try {
+ heartbeatInterval = Time.milliseconds(Duration.apply(strHeartbeatInterval).toMillis());
+ } catch (NumberFormatException e) {
+ throw new ConfigurationException("Could not parse the resource manager's heartbeat interval " +
+ "value " + timeoutOption + '.', e);
+ }
+
+ return new ResourceManagerConfiguration(timeout, heartbeatInterval);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
index f9f55f8..926be0d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java
@@ -22,7 +22,9 @@ import org.apache.flink.runtime.clusterframework.ApplicationStatus;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcService;
/**
@@ -33,10 +35,20 @@ import org.apache.flink.runtime.rpc.RpcService;
*/
public class StandaloneResourceManager extends ResourceManager<ResourceID> {
- public StandaloneResourceManager(RpcService rpcService,
+ public StandaloneResourceManager(
+ RpcService rpcService,
+ ResourceManagerConfiguration resourceManagerConfiguration,
HighAvailabilityServices highAvailabilityServices,
- SlotManagerFactory slotManagerFactory) {
- super(rpcService, highAvailabilityServices, slotManagerFactory);
+ SlotManagerFactory slotManagerFactory,
+ MetricRegistry metricRegistry,
+ FatalErrorHandler fatalErrorHandler) {
+ super(
+ rpcService,
+ resourceManagerConfiguration,
+ highAvailabilityServices,
+ slotManagerFactory,
+ metricRegistry,
+ fatalErrorHandler);
}
@Override
@@ -45,13 +57,6 @@ public class StandaloneResourceManager extends ResourceManager<ResourceID> {
}
@Override
- protected void fatalError(final String message, final Throwable error) {
- log.error("FATAL ERROR IN RESOURCE MANAGER: " + message, error);
- // kill this process
- System.exit(EXIT_CODE_FATAL_ERROR);
- }
-
- @Override
protected void shutDownApplication(ApplicationStatus finalStatus, String optionalDiagnostics) {
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ConfigurationException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ConfigurationException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ConfigurationException.java
new file mode 100644
index 0000000..f081fff
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ConfigurationException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.exceptions;
+
+/**
+ * Base class for configuration related exception which occur when creating a configuration.
+ */
+public class ConfigurationException extends Exception {
+ private static final long serialVersionUID = 3971647332059381556L;
+
+ public ConfigurationException(String message) {
+ super(message);
+ }
+
+ public ConfigurationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ConfigurationException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerException.java
new file mode 100644
index 0000000..6b4d646
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.exceptions;
+
+import org.apache.flink.runtime.resourcemanager.ResourceManager;
+
+/**
+ * Base class for {@link ResourceManager} exceptions.
+ */
+public class ResourceManagerException extends Exception {
+ private static final long serialVersionUID = -5503307426519195160L;
+
+ public ResourceManagerException(String message) {
+ super(message);
+ }
+
+ public ResourceManagerException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ResourceManagerException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerRunner.java
new file mode 100644
index 0000000..1e6f04c
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/exceptions/ResourceManagerRunner.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager.exceptions;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.resourcemanager.ResourceManager;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerConfiguration;
+import org.apache.flink.runtime.resourcemanager.StandaloneResourceManager;
+import org.apache.flink.runtime.resourcemanager.slotmanager.DefaultSlotManager;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
+import org.apache.flink.runtime.rpc.RpcService;
+import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Simple {@link StandaloneResourceManager} runner. It instantiates the resource manager's services
+ * and handles fatal errors by shutting the resource manager down.
+ */
+public class ResourceManagerRunner implements FatalErrorHandler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ResourceManagerRunner.class);
+
+ private final Object lock = new Object();
+
+ private final ResourceManager<?> resourceManager;
+
+ public ResourceManagerRunner(
+ final Configuration configuration,
+ final RpcService rpcService,
+ final HighAvailabilityServices highAvailabilityServices,
+ final MetricRegistry metricRegistry) throws ConfigurationException {
+
+ Preconditions.checkNotNull(configuration);
+ Preconditions.checkNotNull(rpcService);
+ Preconditions.checkNotNull(highAvailabilityServices);
+ Preconditions.checkNotNull(metricRegistry);
+
+ final ResourceManagerConfiguration resourceManagerConfiguration = ResourceManagerConfiguration.fromConfiguration(configuration);
+ final SlotManagerFactory slotManagerFactory = new DefaultSlotManager.Factory();
+
+ this.resourceManager = new StandaloneResourceManager(
+ rpcService,
+ resourceManagerConfiguration,
+ highAvailabilityServices,
+ slotManagerFactory,
+ metricRegistry,
+ this);
+ }
+
+ //-------------------------------------------------------------------------------------
+ // Lifecycle management
+ //-------------------------------------------------------------------------------------
+
+ public void start() throws Exception {
+ resourceManager.start();
+ }
+
+ public void shutDown() throws Exception {
+ shutDownInternally();
+ }
+
+ private void shutDownInternally() throws Exception {
+ synchronized (lock) {
+ resourceManager.shutDown();
+ }
+ }
+
+ //-------------------------------------------------------------------------------------
+ // Fatal error handler
+ //-------------------------------------------------------------------------------------
+
+ @Override
+ public void onFatalError(Throwable exception) {
+ LOG.error("Encountered fatal error.", exception);
+
+ try {
+ shutDownInternally();
+ } catch (Exception e) {
+ LOG.error("Could not properly shut down the resource manager.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/WorkerRegistration.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/WorkerRegistration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/WorkerRegistration.java
index ff28f94..7ee7a1f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/WorkerRegistration.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/registration/WorkerRegistration.java
@@ -19,6 +19,7 @@
package org.apache.flink.runtime.resourcemanager.registration;
import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
+import org.apache.flink.util.Preconditions;
import java.io.Serializable;
@@ -29,11 +30,11 @@ public class WorkerRegistration<WorkerType extends Serializable> extends TaskExe
private static final long serialVersionUID = -2062957799469434614L;
- private WorkerType worker;
+ private final WorkerType worker;
public WorkerRegistration(TaskExecutorGateway taskExecutorGateway, WorkerType worker) {
super(taskExecutorGateway);
- this.worker = worker;
+ this.worker = Preconditions.checkNotNull(worker);
}
public WorkerType getWorker() {
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
index ce1fdca..cb38e6e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java
@@ -18,24 +18,20 @@
package org.apache.flink.runtime.resourcemanager;
+import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
+import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
-import org.apache.flink.runtime.rpc.MainThreadExecutable;
-import org.apache.flink.runtime.rpc.RpcEndpoint;
-import org.apache.flink.runtime.rpc.RpcGateway;
import org.apache.flink.runtime.rpc.RpcService;
-import org.apache.flink.runtime.rpc.StartStoppable;
+import org.apache.flink.runtime.rpc.TestingSerialRpcService;
+import org.apache.flink.runtime.util.TestingFatalErrorHandler;
import org.junit.Assert;
import org.junit.Test;
import java.util.UUID;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
/**
* resourceManager HA test, including grant leadership and revoke leadership
@@ -44,37 +40,38 @@ public class ResourceManagerHATest {
@Test
public void testGrantAndRevokeLeadership() throws Exception {
- // mock a RpcService which will return a special RpcGateway when call its startServer method,
- // the returned RpcGateway directly executes runAsync calls
- TestingResourceManagerGatewayProxy gateway = mock(TestingResourceManagerGatewayProxy.class);
- doCallRealMethod().when(gateway).runAsync(any(Runnable.class));
-
- RpcService rpcService = mock(RpcService.class);
- when(rpcService.startServer(any(RpcEndpoint.class))).thenReturn(gateway);
+ RpcService rpcService = new TestingSerialRpcService();
TestingLeaderElectionService leaderElectionService = new TestingLeaderElectionService();
TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
highAvailabilityServices.setResourceManagerLeaderElectionService(leaderElectionService);
+ ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L));
+ SlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
+ MetricRegistry metricRegistry = mock(MetricRegistry.class);
+ TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
+
final ResourceManager resourceManager =
- new TestingResourceManager(rpcService, highAvailabilityServices);
+ new StandaloneResourceManager(
+ rpcService,
+ resourceManagerConfiguration,
+ highAvailabilityServices,
+ slotManagerFactory,
+ metricRegistry,
+ testingFatalErrorHandler);
resourceManager.start();
// before grant leadership, resourceManager's leaderId is null
- Assert.assertEquals(new UUID(0,0), resourceManager.getLeaderSessionID());
+ Assert.assertEquals(null, resourceManager.getLeaderSessionID());
final UUID leaderId = UUID.randomUUID();
leaderElectionService.isLeader(leaderId);
// after grant leadership, resourceManager's leaderId has value
Assert.assertEquals(leaderId, resourceManager.getLeaderSessionID());
// then revoke leadership, resourceManager's leaderId is null again
leaderElectionService.notLeader();
- Assert.assertEquals(new UUID(0,0), resourceManager.getLeaderSessionID());
- }
+ Assert.assertEquals(null, resourceManager.getLeaderSessionID());
- private static abstract class TestingResourceManagerGatewayProxy implements MainThreadExecutable, StartStoppable, RpcGateway {
- @Override
- public void runAsync(Runnable runnable) {
- runnable.run();
+ if (testingFatalErrorHandler.hasExceptionOccurred()) {
+ testingFatalErrorHandler.rethrowError();
}
}
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
index 14afd0e..7b8d254 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java
@@ -19,14 +19,19 @@
package org.apache.flink.runtime.resourcemanager;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
import org.apache.flink.runtime.jobmaster.JobMasterGateway;
import org.apache.flink.runtime.jobmaster.JobMasterRegistrationSuccess;
import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.TestingSerialRpcService;
import org.apache.flink.runtime.registration.RegistrationResponse;
+import org.apache.flink.runtime.util.TestingFatalErrorHandler;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -61,13 +66,18 @@ public class ResourceManagerJobMasterTest {
TestingLeaderElectionService resourceManagerLeaderElectionService = new TestingLeaderElectionService();
UUID jmLeaderID = UUID.randomUUID();
TestingLeaderRetrievalService jobMasterLeaderRetrievalService = new TestingLeaderRetrievalService(jobMasterAddress, jmLeaderID);
- final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService);
+ TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
+ final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService, testingFatalErrorHandler);
final UUID rmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
// test response successful
Future<RegistrationResponse> successfulFuture = resourceManager.registerJobMaster(rmLeaderSessionId, jmLeaderID, jobMasterAddress, jobID);
RegistrationResponse response = successfulFuture.get(5, TimeUnit.SECONDS);
assertTrue(response instanceof JobMasterRegistrationSuccess);
+
+ if (testingFatalErrorHandler.hasExceptionOccurred()) {
+ testingFatalErrorHandler.rethrowError();
+ }
}
/**
@@ -80,13 +90,18 @@ public class ResourceManagerJobMasterTest {
TestingLeaderElectionService resourceManagerLeaderElectionService = new TestingLeaderElectionService();
UUID jmLeaderID = UUID.randomUUID();
TestingLeaderRetrievalService jobMasterLeaderRetrievalService = new TestingLeaderRetrievalService(jobMasterAddress, jmLeaderID);
- final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService);
+ TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
+ final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService, testingFatalErrorHandler);
final UUID rmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
// test throw exception when receive a registration from job master which takes unmatched leaderSessionId
UUID differentLeaderSessionID = UUID.randomUUID();
Future<RegistrationResponse> unMatchedLeaderFuture = resourceManager.registerJobMaster(differentLeaderSessionID, jmLeaderID, jobMasterAddress, jobID);
assertTrue(unMatchedLeaderFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
+
+ if (testingFatalErrorHandler.hasExceptionOccurred()) {
+ testingFatalErrorHandler.rethrowError();
+ }
}
/**
@@ -98,7 +113,8 @@ public class ResourceManagerJobMasterTest {
JobID jobID = mockJobMaster(jobMasterAddress);
TestingLeaderElectionService resourceManagerLeaderElectionService = new TestingLeaderElectionService();
TestingLeaderRetrievalService jobMasterLeaderRetrievalService = new TestingLeaderRetrievalService();
- final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService);
+ TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
+ final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService, testingFatalErrorHandler);
final UUID rmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
final UUID jmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
@@ -106,6 +122,10 @@ public class ResourceManagerJobMasterTest {
UUID differentLeaderSessionID = UUID.randomUUID();
Future<RegistrationResponse> unMatchedLeaderFuture = resourceManager.registerJobMaster(rmLeaderSessionId, differentLeaderSessionID, jobMasterAddress, jobID);
assertTrue(unMatchedLeaderFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
+
+ if (testingFatalErrorHandler.hasExceptionOccurred()) {
+ testingFatalErrorHandler.rethrowError();
+ }
}
/**
@@ -117,7 +137,8 @@ public class ResourceManagerJobMasterTest {
JobID jobID = mockJobMaster(jobMasterAddress);
TestingLeaderElectionService resourceManagerLeaderElectionService = new TestingLeaderElectionService();
TestingLeaderRetrievalService jobMasterLeaderRetrievalService = new TestingLeaderRetrievalService();
- final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService);
+ TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
+ final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService, testingFatalErrorHandler);
final UUID rmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
final UUID jmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
@@ -125,6 +146,10 @@ public class ResourceManagerJobMasterTest {
String invalidAddress = "/jobMasterAddress2";
Future<RegistrationResponse> invalidAddressFuture = resourceManager.registerJobMaster(rmLeaderSessionId, jmLeaderSessionId, invalidAddress, jobID);
assertTrue(invalidAddressFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
+
+ if (testingFatalErrorHandler.hasExceptionOccurred()) {
+ testingFatalErrorHandler.rethrowError();
+ }
}
/**
@@ -136,7 +161,8 @@ public class ResourceManagerJobMasterTest {
JobID jobID = mockJobMaster(jobMasterAddress);
TestingLeaderElectionService resourceManagerLeaderElectionService = new TestingLeaderElectionService();
TestingLeaderRetrievalService jobMasterLeaderRetrievalService = new TestingLeaderRetrievalService();
- final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService);
+ TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
+ final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService, testingFatalErrorHandler);
final UUID rmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
final UUID jmLeaderSessionId = grantResourceManagerLeadership(resourceManagerLeaderElectionService);
@@ -145,6 +171,10 @@ public class ResourceManagerJobMasterTest {
Future<RegistrationResponse> declineFuture = resourceManager.registerJobMaster(rmLeaderSessionId, jmLeaderSessionId, jobMasterAddress, unknownJobIDToHAServices);
RegistrationResponse response = declineFuture.get(5, TimeUnit.SECONDS);
assertTrue(response instanceof RegistrationResponse.Decline);
+
+ if (testingFatalErrorHandler.hasExceptionOccurred()) {
+ testingFatalErrorHandler.rethrowError();
+ }
}
private JobID mockJobMaster(String jobMasterAddress) {
@@ -154,11 +184,26 @@ public class ResourceManagerJobMasterTest {
return jobID;
}
- private ResourceManager createAndStartResourceManager(TestingLeaderElectionService resourceManagerLeaderElectionService, JobID jobID, TestingLeaderRetrievalService jobMasterLeaderRetrievalService) {
+ private ResourceManager createAndStartResourceManager(
+ TestingLeaderElectionService resourceManagerLeaderElectionService,
+ JobID jobID,
+ TestingLeaderRetrievalService jobMasterLeaderRetrievalService,
+ FatalErrorHandler fatalErrorHandler) throws Exception {
TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
highAvailabilityServices.setResourceManagerLeaderElectionService(resourceManagerLeaderElectionService);
highAvailabilityServices.setJobMasterLeaderRetriever(jobID, jobMasterLeaderRetrievalService);
- ResourceManager resourceManager = new TestingResourceManager(rpcService, highAvailabilityServices);
+
+ ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L));
+ SlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
+ MetricRegistry metricRegistry = mock(MetricRegistry.class);
+
+ ResourceManager resourceManager = new StandaloneResourceManager(
+ rpcService,
+ resourceManagerConfiguration,
+ highAvailabilityServices,
+ slotManagerFactory,
+ metricRegistry,
+ fatalErrorHandler);
resourceManager.start();
return resourceManager;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
index a577c26..4640eab 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java
@@ -18,15 +18,19 @@
package org.apache.flink.runtime.resourcemanager;
+import org.apache.flink.api.common.time.Time;
import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.concurrent.Future;
import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.TestingSerialRpcService;
import org.apache.flink.runtime.registration.RegistrationResponse;
import org.apache.flink.runtime.taskexecutor.SlotReport;
import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway;
import org.apache.flink.runtime.taskexecutor.TaskExecutorRegistrationSuccess;
+import org.apache.flink.runtime.util.TestingFatalErrorHandler;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -34,7 +38,6 @@ import org.junit.Test;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -53,13 +56,16 @@ public class ResourceManagerTaskExecutorTest {
private UUID leaderSessionId;
+ private TestingFatalErrorHandler testingFatalErrorHandler;
+
@Before
public void setup() throws Exception {
rpcService = new TestingSerialRpcService();
taskExecutorResourceID = mockTaskExecutor(taskExecutorAddress);
TestingLeaderElectionService rmLeaderElectionService = new TestingLeaderElectionService();
- resourceManager = createAndStartResourceManager(rmLeaderElectionService);
+ testingFatalErrorHandler = new TestingFatalErrorHandler();
+ resourceManager = createAndStartResourceManager(rmLeaderElectionService, testingFatalErrorHandler);
leaderSessionId = grantLeadership(rmLeaderElectionService);
}
@@ -73,18 +79,24 @@ public class ResourceManagerTaskExecutorTest {
*/
@Test
public void testRegisterTaskExecutor() throws Exception {
- // test response successful
- Future<RegistrationResponse> successfulFuture =
- resourceManager.registerTaskExecutor(leaderSessionId, taskExecutorAddress, taskExecutorResourceID, slotReport);
- RegistrationResponse response = successfulFuture.get(5, TimeUnit.SECONDS);
- assertTrue(response instanceof TaskExecutorRegistrationSuccess);
-
- // test response successful with instanceID not equal to previous when receive duplicate registration from taskExecutor
- Future<RegistrationResponse> duplicateFuture =
- resourceManager.registerTaskExecutor(leaderSessionId, taskExecutorAddress, taskExecutorResourceID, slotReport);
- RegistrationResponse duplicateResponse = duplicateFuture.get();
- assertTrue(duplicateResponse instanceof TaskExecutorRegistrationSuccess);
- assertNotEquals(((TaskExecutorRegistrationSuccess) response).getRegistrationId(), ((TaskExecutorRegistrationSuccess) duplicateResponse).getRegistrationId());
+ try {
+ // test response successful
+ Future<RegistrationResponse> successfulFuture =
+ resourceManager.registerTaskExecutor(leaderSessionId, taskExecutorAddress, taskExecutorResourceID, slotReport);
+ RegistrationResponse response = successfulFuture.get(5, TimeUnit.SECONDS);
+ assertTrue(response instanceof TaskExecutorRegistrationSuccess);
+
+ // test response successful with instanceID not equal to previous when receive duplicate registration from taskExecutor
+ Future<RegistrationResponse> duplicateFuture =
+ resourceManager.registerTaskExecutor(leaderSessionId, taskExecutorAddress, taskExecutorResourceID, slotReport);
+ RegistrationResponse duplicateResponse = duplicateFuture.get();
+ assertTrue(duplicateResponse instanceof TaskExecutorRegistrationSuccess);
+ assertNotEquals(((TaskExecutorRegistrationSuccess) response).getRegistrationId(), ((TaskExecutorRegistrationSuccess) duplicateResponse).getRegistrationId());
+ } finally {
+ if (testingFatalErrorHandler.hasExceptionOccurred()) {
+ testingFatalErrorHandler.rethrowError();
+ }
+ }
}
/**
@@ -92,11 +104,17 @@ public class ResourceManagerTaskExecutorTest {
*/
@Test
public void testRegisterTaskExecutorWithUnmatchedLeaderSessionId() throws Exception {
- // test throw exception when receive a registration from taskExecutor which takes unmatched leaderSessionId
- UUID differentLeaderSessionID = UUID.randomUUID();
- Future<RegistrationResponse> unMatchedLeaderFuture =
- resourceManager.registerTaskExecutor(differentLeaderSessionID, taskExecutorAddress, taskExecutorResourceID, slotReport);
- assertTrue(unMatchedLeaderFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
+ try {
+ // test throw exception when receive a registration from taskExecutor which takes unmatched leaderSessionId
+ UUID differentLeaderSessionID = UUID.randomUUID();
+ Future<RegistrationResponse> unMatchedLeaderFuture =
+ resourceManager.registerTaskExecutor(differentLeaderSessionID, taskExecutorAddress, taskExecutorResourceID, slotReport);
+ assertTrue(unMatchedLeaderFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
+ } finally {
+ if (testingFatalErrorHandler.hasExceptionOccurred()) {
+ testingFatalErrorHandler.rethrowError();
+ }
+ }
}
/**
@@ -104,11 +122,17 @@ public class ResourceManagerTaskExecutorTest {
*/
@Test
public void testRegisterTaskExecutorFromInvalidAddress() throws Exception {
- // test throw exception when receive a registration from taskExecutor which takes invalid address
- String invalidAddress = "/taskExecutor2";
- Future<RegistrationResponse> invalidAddressFuture =
- resourceManager.registerTaskExecutor(leaderSessionId, invalidAddress, taskExecutorResourceID, slotReport);
- assertTrue(invalidAddressFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
+ try {
+ // test throw exception when receive a registration from taskExecutor which takes invalid address
+ String invalidAddress = "/taskExecutor2";
+ Future<RegistrationResponse> invalidAddressFuture =
+ resourceManager.registerTaskExecutor(leaderSessionId, invalidAddress, taskExecutorResourceID, slotReport);
+ assertTrue(invalidAddressFuture.get(5, TimeUnit.SECONDS) instanceof RegistrationResponse.Decline);
+ } finally {
+ if (testingFatalErrorHandler.hasExceptionOccurred()) {
+ testingFatalErrorHandler.rethrowError();
+ }
+ }
}
private ResourceID mockTaskExecutor(String taskExecutorAddress) {
@@ -118,11 +142,22 @@ public class ResourceManagerTaskExecutorTest {
return taskExecutorResourceID;
}
- private StandaloneResourceManager createAndStartResourceManager(TestingLeaderElectionService rmLeaderElectionService) {
+ private StandaloneResourceManager createAndStartResourceManager(TestingLeaderElectionService rmLeaderElectionService, FatalErrorHandler fatalErrorHandler) throws Exception {
TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
+ TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
+ ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L));
+ MetricRegistry metricRegistry = mock(MetricRegistry.class);
+
+
StandaloneResourceManager resourceManager =
- new TestingResourceManager(rpcService, highAvailabilityServices);
+ new StandaloneResourceManager(
+ rpcService,
+ resourceManagerConfiguration,
+ highAvailabilityServices,
+ slotManagerFactory,
+ metricRegistry,
+ fatalErrorHandler);
resourceManager.start();
return resourceManager;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java
deleted file mode 100644
index 6b4ca14..0000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.runtime.resourcemanager;
-
-import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
-import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
-import org.apache.flink.runtime.rpc.RpcService;
-
-public class TestingResourceManager extends StandaloneResourceManager {
-
- public TestingResourceManager(RpcService rpcService) {
- this(rpcService, new TestingHighAvailabilityServices());
- }
-
- public TestingResourceManager(
- RpcService rpcService,
- HighAvailabilityServices highAvailabilityServices) {
- this(rpcService, highAvailabilityServices, new TestingSlotManagerFactory());
- }
-
- public TestingResourceManager(
- RpcService rpcService,
- HighAvailabilityServices highAvailabilityServices,
- SlotManagerFactory slotManagerFactory) {
- super(rpcService, highAvailabilityServices, slotManagerFactory);
- }
-
- private static class TestingSlotManagerFactory implements SlotManagerFactory {
-
- @Override
- public SlotManager create(ResourceManagerServices rmServices) {
- return new TestingSlotManager(rmServices);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManagerFactory.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManagerFactory.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManagerFactory.java
new file mode 100644
index 0000000..6b5f6b2
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingSlotManagerFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.resourcemanager;
+
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager;
+import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerFactory;
+
+public class TestingSlotManagerFactory implements SlotManagerFactory {
+
+ @Override
+ public SlotManager create(ResourceManagerServices rmServices) {
+ return new TestingSlotManager(rmServices);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
index 86cd1f8..08ceb86 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotProtocolTest.java
@@ -30,14 +30,18 @@ import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices
import org.apache.flink.runtime.jobmaster.JobMasterGateway;
import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService;
import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService;
+import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.registration.RegistrationResponse;
+import org.apache.flink.runtime.resourcemanager.ResourceManager;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerConfiguration;
import org.apache.flink.runtime.resourcemanager.ResourceManagerServices;
import org.apache.flink.runtime.resourcemanager.SlotRequest;
-import org.apache.flink.runtime.resourcemanager.TestingResourceManager;
+import org.apache.flink.runtime.resourcemanager.StandaloneResourceManager;
import org.apache.flink.runtime.resourcemanager.TestingSlotManager;
import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestReply;
import org.apache.flink.runtime.resourcemanager.messages.taskexecutor.TMSlotRequestReply;
import org.apache.flink.runtime.resourcemanager.registration.TaskExecutorRegistration;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.RpcService;
import org.apache.flink.runtime.rpc.TestingSerialRpcService;
import org.apache.flink.runtime.taskexecutor.SlotReport;
@@ -102,9 +106,17 @@ public class SlotProtocolTest extends TestLogger {
TestingLeaderElectionService rmLeaderElectionService =
configureHA(testingHaServices, jobID, rmAddress, rmLeaderID, jmAddress, jmLeaderID);
+ ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L));
+
final TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
SpiedResourceManager resourceManager =
- new SpiedResourceManager(testRpcService, testingHaServices, slotManagerFactory);
+ new SpiedResourceManager(
+ testRpcService,
+ resourceManagerConfiguration,
+ testingHaServices,
+ slotManagerFactory,
+ mock(MetricRegistry.class),
+ mock(FatalErrorHandler.class));
resourceManager.start();
rmLeaderElectionService.isLeader(rmLeaderID);
@@ -193,16 +205,26 @@ public class SlotProtocolTest extends TestLogger {
.thenReturn(new FlinkCompletableFuture<TMSlotRequestReply>());
testRpcService.registerGateway(tmAddress, taskExecutorGateway);
+ ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration(Time.seconds(5L), Time.seconds(5L));
+
TestingSlotManagerFactory slotManagerFactory = new TestingSlotManagerFactory();
- TestingResourceManager resourceManager =
- Mockito.spy(new TestingResourceManager(testRpcService, testingHaServices, slotManagerFactory));
+ ResourceManager<ResourceID> resourceManager =
+ Mockito.spy(new StandaloneResourceManager(
+ testRpcService,
+ resourceManagerConfiguration,
+ testingHaServices,
+ slotManagerFactory,
+ mock(MetricRegistry.class),
+ mock(FatalErrorHandler.class)));
resourceManager.start();
rmLeaderElectionService.isLeader(rmLeaderID);
+ Thread.sleep(1000);
+
Future<RegistrationResponse> registrationFuture =
resourceManager.registerJobMaster(rmLeaderID, jmLeaderID, jmAddress, jobID);
try {
- registrationFuture.get(5, TimeUnit.SECONDS);
+ registrationFuture.get(5L, TimeUnit.SECONDS);
} catch (Exception e) {
Assert.fail("JobManager registration Future didn't become ready.");
}
@@ -258,15 +280,24 @@ public class SlotProtocolTest extends TestLogger {
return rmLeaderElectionService;
}
- private static class SpiedResourceManager extends TestingResourceManager {
+ private static class SpiedResourceManager extends StandaloneResourceManager {
private int startNewWorkerCalled = 0;
public SpiedResourceManager(
RpcService rpcService,
+ ResourceManagerConfiguration resourceManagerConfiguration,
HighAvailabilityServices highAvailabilityServices,
- SlotManagerFactory slotManagerFactory) {
- super(rpcService, highAvailabilityServices, slotManagerFactory);
+ SlotManagerFactory slotManagerFactory,
+ MetricRegistry metricRegistry,
+ FatalErrorHandler fatalErrorHandler) {
+ super(
+ rpcService,
+ resourceManagerConfiguration,
+ highAvailabilityServices,
+ slotManagerFactory,
+ metricRegistry,
+ fatalErrorHandler);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7fab3edf/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index caae54e..553db37 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -65,12 +65,12 @@ import org.apache.flink.runtime.rpc.FatalErrorHandler;
import org.apache.flink.runtime.rpc.TestingSerialRpcService;
import org.apache.flink.runtime.taskexecutor.slot.SlotOffer;
import org.apache.flink.runtime.taskexecutor.slot.TaskSlotTable;
-import org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException;
import org.apache.flink.runtime.taskexecutor.slot.TimerService;
import org.apache.flink.runtime.taskmanager.CheckpointResponder;
import org.apache.flink.runtime.taskmanager.Task;
import org.apache.flink.runtime.taskmanager.TaskManagerActions;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
+import org.apache.flink.runtime.util.TestingFatalErrorHandler;
import org.apache.flink.util.SerializedValue;
import org.apache.flink.util.TestLogger;
@@ -81,15 +81,12 @@ import org.junit.Test;
import org.junit.rules.TestName;
import org.mockito.Matchers;
import org.powermock.api.mockito.PowerMockito;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.net.InetAddress;
import java.net.URL;
import java.util.Arrays;
import java.util.Collections;
import java.util.UUID;
-import java.util.concurrent.atomic.AtomicReference;
import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
@@ -434,7 +431,7 @@ public class TaskExecutorTest extends TestLogger {
any(Time.class));
} finally {
// check if a concurrent error occurred
- testingFatalErrorHandler.rethrowException();
+ testingFatalErrorHandler.rethrowError();
rpc.stopService();
}
@@ -541,63 +538,12 @@ public class TaskExecutorTest extends TestLogger {
assertTrue(taskSlotTable.isSlotFree(1));
} finally {
// check if a concurrent error occurred
- testingFatalErrorHandler.rethrowException();
+ testingFatalErrorHandler.rethrowError();
rpc.stopService();
}
}
- private static class TestingFatalErrorHandler implements FatalErrorHandler {
- private static final Logger LOG = LoggerFactory.getLogger(TestingFatalErrorHandler.class);
- private final AtomicReference<Throwable> atomicThrowable;
-
- public TestingFatalErrorHandler() {
- atomicThrowable = new AtomicReference<>(null);
- }
-
- public void rethrowException() throws TestingException {
- Throwable throwable = atomicThrowable.get();
-
- if (throwable != null) {
- throw new TestingException(throwable);
- }
- }
-
- public boolean hasExceptionOccurred() {
- return atomicThrowable.get() != null;
- }
-
- public Throwable getException() {
- return atomicThrowable.get();
- }
-
- @Override
- public void onFatalError(Throwable exception) {
- LOG.error("OnFatalError:", exception);
- atomicThrowable.compareAndSet(null, exception);
- }
-
- //------------------------------------------------------------------
- // static utility classes
- //------------------------------------------------------------------
-
- private static final class TestingException extends Exception {
- public TestingException(String message) {
- super(message);
- }
-
- public TestingException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public TestingException(Throwable cause) {
- super(cause);
- }
-
- private static final long serialVersionUID = -4648195335470914498L;
- }
- }
-
/**
* Tests that all allocation requests for slots are ignored if the slot has been reported as
* free by the TaskExecutor but this report hasn't been confirmed by the ResourceManager.
[39/50] [abbrv] flink git commit: [hotfix] Treat taskManager's rpc
address and location separately
Posted by tr...@apache.org.
[hotfix] Treat taskManager's rpc address and location separately
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e9abc8df
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e9abc8df
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e9abc8df
Branch: refs/heads/flip-6
Commit: e9abc8dffcdb155413997fd26961b5ad4337470a
Parents: 60ee943
Author: Kurt Young <yk...@gmail.com>
Authored: Mon Oct 17 09:38:46 2016 +0800
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:33 2016 +0100
----------------------------------------------------------------------
.../flink/runtime/jobmaster/JobMaster.java | 7 ++--
.../runtime/jobmaster/JobMasterGateway.java | 8 +++--
.../runtime/taskexecutor/JobLeaderService.java | 37 +++++++++++++-------
.../runtime/taskexecutor/TaskExecutor.java | 2 +-
.../runtime/taskexecutor/TaskExecutorTest.java | 14 ++++----
5 files changed, 43 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e9abc8df/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
index 8cb9946..306a28a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java
@@ -680,13 +680,14 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
@RpcMethod
public Future<RegistrationResponse> registerTaskManager(
+ final String taskManagerRpcAddress,
final TaskManagerLocation taskManagerLocation,
final UUID leaderId) throws Exception
{
if (!JobMaster.this.leaderSessionID.equals(leaderId)) {
log.warn("Discard registration from TaskExecutor {} at ({}) because the expected " +
"leader session ID {} did not equal the received leader session ID {}.",
- taskManagerLocation.getResourceID(), taskManagerLocation.addressString(),
+ taskManagerLocation.getResourceID(), taskManagerRpcAddress,
JobMaster.this.leaderSessionID, leaderId);
throw new Exception("Leader id not match, expected: " + JobMaster.this.leaderSessionID
+ ", actual: " + leaderId);
@@ -702,7 +703,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
return getRpcService().execute(new Callable<TaskExecutorGateway>() {
@Override
public TaskExecutorGateway call() throws Exception {
- return getRpcService().connect(taskManagerLocation.addressString(), TaskExecutorGateway.class)
+ return getRpcService().connect(taskManagerRpcAddress, TaskExecutorGateway.class)
.get(rpcTimeout.getSize(), rpcTimeout.getUnit());
}
}).handleAsync(new BiFunction<TaskExecutorGateway, Throwable, RegistrationResponse>() {
@@ -715,7 +716,7 @@ public class JobMaster extends RpcEndpoint<JobMasterGateway> {
if (!JobMaster.this.leaderSessionID.equals(leaderId)) {
log.warn("Discard registration from TaskExecutor {} at ({}) because the expected " +
"leader session ID {} did not equal the received leader session ID {}.",
- taskManagerLocation.getResourceID(), taskManagerLocation.addressString(),
+ taskManagerId, taskManagerRpcAddress,
JobMaster.this.leaderSessionID, leaderId);
return new RegistrationResponse.Decline("Invalid leader session id");
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e9abc8df/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
index 4c85839..4ee9f92 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java
@@ -196,12 +196,14 @@ public interface JobMasterGateway extends CheckpointCoordinatorGateway {
/**
* Register the task manager at the job manager.
*
- * @param taskManagerLocation location of the task manager
- * @param leaderId identifying the job leader
- * @param timeout for the rpc call
+ * @param taskManagerRpcAddress the rpc address of the task manager
+ * @param taskManagerLocation location of the task manager
+ * @param leaderId identifying the job leader
+ * @param timeout for the rpc call
* @return Future registration response indicating whether the registration was successful or not
*/
Future<RegistrationResponse> registerTaskManager(
+ final String taskManagerRpcAddress,
final TaskManagerLocation taskManagerLocation,
final UUID leaderId,
@RpcTimeout final Time timeout);
http://git-wip-us.apache.org/repos/asf/flink/blob/e9abc8df/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
index 14d36ab..93c7bb7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java
@@ -62,6 +62,9 @@ public class JobLeaderService {
/** Internal state of the service */
private volatile JobLeaderService.State state;
+ /** Address of the owner of this service. This address is used for the job manager connection */
+ private String ownerAddress;
+
/** Rpc service to use for establishing connections */
private RpcService rpcService;
@@ -78,6 +81,7 @@ public class JobLeaderService {
state = JobLeaderService.State.CREATED;
+ ownerAddress = null;
rpcService = null;
highAvailabilityServices = null;
jobLeaderListener = null;
@@ -90,20 +94,23 @@ public class JobLeaderService {
/**
* Start the job leader service with the given services.
*
+ * @param initialOwnerAddress to be used for establishing connections (source address)
* @param initialRpcService to be used to create rpc connections
* @param initialHighAvailabilityServices to create leader retrieval services for the different jobs
* @param initialJobLeaderListener listening for job leader changes
*/
public void start(
- final RpcService initialRpcService,
- final HighAvailabilityServices initialHighAvailabilityServices,
- final JobLeaderListener initialJobLeaderListener) {
+ final String initialOwnerAddress,
+ final RpcService initialRpcService,
+ final HighAvailabilityServices initialHighAvailabilityServices,
+ final JobLeaderListener initialJobLeaderListener) {
if (JobLeaderService.State.CREATED != state) {
throw new IllegalStateException("The service has already been started.");
} else {
LOG.info("Start job leader service.");
+ this.ownerAddress = Preconditions.checkNotNull(initialOwnerAddress);
this.rpcService = Preconditions.checkNotNull(initialRpcService);
this.highAvailabilityServices = Preconditions.checkNotNull(initialHighAvailabilityServices);
this.jobLeaderListener = Preconditions.checkNotNull(initialJobLeaderListener);
@@ -310,6 +317,7 @@ public class JobLeaderService {
JobMasterGateway.class,
getTargetAddress(),
getTargetLeaderId(),
+ ownerAddress,
ownLocation);
}
@@ -345,19 +353,23 @@ public class JobLeaderService {
extends RetryingRegistration<JobMasterGateway, JMTMRegistrationSuccess>
{
+ private final String taskManagerRpcAddress;
+
private final TaskManagerLocation taskManagerLocation;
JobManagerRetryingRegistration(
- Logger log,
- RpcService rpcService,
- String targetName,
- Class<JobMasterGateway> targetType,
- String targetAddress,
- UUID leaderId,
- TaskManagerLocation taskManagerLocation) {
-
+ Logger log,
+ RpcService rpcService,
+ String targetName,
+ Class<JobMasterGateway> targetType,
+ String targetAddress,
+ UUID leaderId,
+ String taskManagerRpcAddress,
+ TaskManagerLocation taskManagerLocation)
+ {
super(log, rpcService, targetName, targetType, targetAddress, leaderId);
+ this.taskManagerRpcAddress = taskManagerRpcAddress;
this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
}
@@ -365,7 +377,8 @@ public class JobLeaderService {
protected Future<RegistrationResponse> invokeRegistration(
JobMasterGateway gateway, UUID leaderId, long timeoutMillis) throws Exception
{
- return gateway.registerTaskManager(taskManagerLocation, leaderId, Time.milliseconds(timeoutMillis));
+ return gateway.registerTaskManager(taskManagerRpcAddress, taskManagerLocation,
+ leaderId, Time.milliseconds(timeoutMillis));
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e9abc8df/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index 1201281..3e3a544 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -206,7 +206,7 @@ public class TaskExecutor extends RpcEndpoint<TaskExecutorGateway> {
taskSlotTable.start(new SlotActionsImpl());
// start the job leader service
- jobLeaderService.start(getRpcService(), haServices, new JobLeaderListenerImpl());
+ jobLeaderService.start(getAddress(), getRpcService(), haServices, new JobLeaderListenerImpl());
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/e9abc8df/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index 2220f12..2b5d2dd 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -379,9 +379,10 @@ public class TaskExecutorTest extends TestLogger {
final JobMasterGateway jobMasterGateway = mock(JobMasterGateway.class);
when(jobMasterGateway.registerTaskManager(
- eq(taskManagerLocation),
- eq(jobManagerLeaderId),
- any(Time.class)
+ any(String.class),
+ eq(taskManagerLocation),
+ eq(jobManagerLeaderId),
+ any(Time.class)
)).thenReturn(FlinkCompletableFuture.<RegistrationResponse>completed(new JMTMRegistrationSuccess(jmResourceId, blobPort)));
when(jobMasterGateway.getAddress()).thenReturn(jobManagerAddress);
@@ -483,9 +484,10 @@ public class TaskExecutorTest extends TestLogger {
final JobMasterGateway jobMasterGateway = mock(JobMasterGateway.class);
when(jobMasterGateway.registerTaskManager(
- eq(taskManagerLocation),
- eq(jobManagerLeaderId),
- any(Time.class)
+ any(String.class),
+ eq(taskManagerLocation),
+ eq(jobManagerLeaderId),
+ any(Time.class)
)).thenReturn(FlinkCompletableFuture.<RegistrationResponse>completed(new JMTMRegistrationSuccess(jmResourceId, blobPort)));
when(jobMasterGateway.getAddress()).thenReturn(jobManagerAddress);
[30/50] [abbrv] flink git commit: [FLINK-4339] [cluster management]
Implement Slot Pool core on JobManager side
Posted by tr...@apache.org.
[FLINK-4339] [cluster management] Implement Slot Pool core on JobManager side
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1801de9c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1801de9c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1801de9c
Branch: refs/heads/flip-6
Commit: 1801de9c669e7e6be514d8d4ad9e6d8c95617573
Parents: 5f62d3b
Author: Kurt Young <yk...@gmail.com>
Authored: Thu Oct 13 04:59:46 2016 +0800
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Nov 1 09:39:31 2016 +0100
----------------------------------------------------------------------
.../runtime/clusterframework/types/SlotID.java | 16 +-
.../flink/runtime/instance/SlotDescriptor.java | 161 +++++
.../apache/flink/runtime/instance/SlotPool.java | 675 +++++++++++++++++++
.../apache/flink/runtime/akka/AkkaUtils.scala | 4 +-
.../runtime/instance/AllocatedSlotsTest.java | 135 ++++
.../runtime/instance/AvailableSlotsTest.java | 123 ++++
.../flink/runtime/instance/SlotPoolTest.java | 297 ++++++++
7 files changed, 1403 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1801de9c/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
index e831a5d..237597b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotID.java
@@ -33,11 +33,11 @@ public class SlotID implements ResourceIDRetrievable, Serializable {
private final ResourceID resourceId;
/** The numeric id for single slot */
- private final int slotId;
+ private final int slotNumber;
- public SlotID(ResourceID resourceId, int slotId) {
+ public SlotID(ResourceID resourceId, int slotNumber) {
this.resourceId = checkNotNull(resourceId, "ResourceID must not be null");
- this.slotId = slotId;
+ this.slotNumber = slotNumber;
}
// ------------------------------------------------------------------------
@@ -47,6 +47,10 @@ public class SlotID implements ResourceIDRetrievable, Serializable {
return resourceId;
}
+ public int getSlotNumber() {
+ return slotNumber;
+ }
+
// ------------------------------------------------------------------------
@Override
@@ -60,7 +64,7 @@ public class SlotID implements ResourceIDRetrievable, Serializable {
SlotID slotID = (SlotID) o;
- if (slotId != slotID.slotId) {
+ if (slotNumber != slotID.slotNumber) {
return false;
}
return resourceId.equals(slotID.resourceId);
@@ -69,13 +73,13 @@ public class SlotID implements ResourceIDRetrievable, Serializable {
@Override
public int hashCode() {
int result = resourceId.hashCode();
- result = 31 * result + slotId;
+ result = 31 * result + slotNumber;
return result;
}
@Override
public String toString() {
- return resourceId + "_" + slotId;
+ return resourceId + "_" + slotNumber;
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/1801de9c/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotDescriptor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotDescriptor.java
new file mode 100644
index 0000000..be7cf96
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotDescriptor.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.instance;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * The description of slots, TaskManagers offer one or more task slots, which define a slice of
+ * their resources. This description will contain some static information about the slot, such
+ * as the location and numeric id of the slot, rpc gateway to communicate with the TaskManager which
+ * owns the slot.
+ */
+public class SlotDescriptor {
+
+ /** The ID of the job this slice belongs to. */
+ private final JobID jobID;
+
+ /** The location information of the TaskManager to which this slot belongs */
+ private final TaskManagerLocation taskManagerLocation;
+
+ /** The number of the slot on which the task is deployed */
+ private final int slotNumber;
+
+ /** The resource profile of the slot provides */
+ private final ResourceProfile resourceProfile;
+
+ /** TEMP until the new RPC is in place: The actor gateway to communicate with the TaskManager */
+ private final ActorGateway taskManagerActorGateway;
+
+ public SlotDescriptor(
+ final JobID jobID,
+ final TaskManagerLocation location,
+ final int slotNumber,
+ final ResourceProfile resourceProfile,
+ final ActorGateway actorGateway)
+ {
+ this.jobID = checkNotNull(jobID);
+ this.taskManagerLocation = checkNotNull(location);
+ this.slotNumber = slotNumber;
+ this.resourceProfile = checkNotNull(resourceProfile);
+ this.taskManagerActorGateway = checkNotNull(actorGateway);
+ }
+
+ public SlotDescriptor(final SlotDescriptor other) {
+ this.jobID = other.jobID;
+ this.taskManagerLocation = other.taskManagerLocation;
+ this.slotNumber = other.slotNumber;
+ this.resourceProfile = other.resourceProfile;
+ this.taskManagerActorGateway = other.taskManagerActorGateway;
+ }
+
+ // TODO - temporary workaround until we have the SlotDesriptor in the Slot
+ public SlotDescriptor(final Slot slot) {
+ this.jobID = slot.getJobID();
+ this.taskManagerLocation = slot.getTaskManagerLocation();
+ this.slotNumber = slot.getRootSlotNumber();
+ this.resourceProfile = new ResourceProfile(0, 0);
+ this.taskManagerActorGateway = slot.getTaskManagerActorGateway();
+ }
+
+ /**
+ * Returns the ID of the job this allocated slot belongs to.
+ *
+ * @return the ID of the job this allocated slot belongs to
+ */
+ public JobID getJobID() {
+ return jobID;
+ }
+
+ /**
+ * Gets the number of the slot.
+ *
+ * @return The number of the slot on the TaskManager.
+ */
+ public int getSlotNumber() {
+ return slotNumber;
+ }
+
+ /**
+ * Gets the resource profile of the slot.
+ *
+ * @return The resource profile of the slot.
+ */
+ public ResourceProfile getResourceProfile() {
+ return resourceProfile;
+ }
+
+ /**
+ * Gets the location info of the TaskManager that offers this slot.
+ *
+ * @return The location info of the TaskManager that offers this slot
+ */
+ public TaskManagerLocation getTaskManagerLocation() {
+ return taskManagerLocation;
+ }
+
+ /**
+ * Gets the actor gateway that can be used to send messages to the TaskManager.
+ * <p>
+ * This method should be removed once the new interface-based RPC abstraction is in place
+ *
+ * @return The actor gateway that can be used to send messages to the TaskManager.
+ */
+ public ActorGateway getTaskManagerActorGateway() {
+ return taskManagerActorGateway;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ SlotDescriptor that = (SlotDescriptor) o;
+
+ if (slotNumber != that.slotNumber) {
+ return false;
+ }
+ if (!jobID.equals(that.jobID)) {
+ return false;
+ }
+ return taskManagerLocation.equals(that.taskManagerLocation);
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = jobID.hashCode();
+ result = 31 * result + taskManagerLocation.hashCode();
+ result = 31 * result + slotNumber;
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return taskManagerLocation + " - " + slotNumber;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1801de9c/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
new file mode 100644
index 0000000..e7857c1
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotPool.java
@@ -0,0 +1,675 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.instance;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import org.apache.flink.runtime.concurrent.ApplyFunction;
+import org.apache.flink.runtime.concurrent.BiFunction;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.concurrent.impl.FlinkCompletableFuture;
+import org.apache.flink.runtime.jobmanager.slots.SlotOwner;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
+import org.apache.flink.runtime.resourcemanager.SlotRequest;
+import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestRejected;
+import org.apache.flink.runtime.resourcemanager.messages.jobmanager.RMSlotRequestReply;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * The slot pool serves slot request issued by Scheduler or ExecutionGraph. It will will attempt to acquire new slots
+ * from the ResourceManager when it cannot serve a slot request. If no ResourceManager is currently available,
+ * or it gets a decline from the ResourceManager, or a request times out, it fails the slot request. The slot pool also
+ * holds all the slots that were offered to it and accepted, and can thus provides registered free slots even if the
+ * ResourceManager is down. The slots will only be released when they are useless, e.g. when the job is fully running
+ * but we still have some free slots.
+ * <p>
+ * All the allocation or the slot offering will be identified by self generated AllocationID, we will use it to
+ * eliminate ambiguities.
+ */
+public class SlotPool implements SlotOwner {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SlotPool.class);
+
+ private final Object lock = new Object();
+
+ /** The executor which is used to execute futures */
+ private final Executor executor;
+
+ /** All registered resources, slots will be accepted and used only if the resource is registered */
+ private final Set<ResourceID> registeredResources;
+
+ /** The book-keeping of all allocated slots */
+ private final AllocatedSlots allocatedSlots;
+
+ /** The book-keeping of all available slots */
+ private final AvailableSlots availableSlots;
+
+ /** All pending requests waiting for slots */
+ private final Map<AllocationID, Tuple2<SlotRequest, FlinkCompletableFuture<SlotDescriptor>>> pendingRequests;
+
+ /** Timeout of slot allocation */
+ private final Time timeout;
+
+ /** the leader id of job manager */
+ private UUID jobManagerLeaderId;
+
+ /** The leader id of resource manager */
+ private UUID resourceManagerLeaderId;
+
+ /** The gateway to communicate with resource manager */
+ private ResourceManagerGateway resourceManagerGateway;
+
+ public SlotPool(final Executor executor) {
+ this.executor = executor;
+ this.registeredResources = new HashSet<>();
+ this.allocatedSlots = new AllocatedSlots();
+ this.availableSlots = new AvailableSlots();
+ this.pendingRequests = new HashMap<>();
+ this.timeout = Time.of(5, TimeUnit.SECONDS);
+ }
+
+ public void setJobManagerLeaderId(final UUID jobManagerLeaderId) {
+ this.jobManagerLeaderId = jobManagerLeaderId;
+ }
+
+ // ------------------------------------------------------------------------
+ // Slot Allocation
+ // ------------------------------------------------------------------------
+
+ /**
+ * Try to allocate a simple slot with specified resource profile.
+ *
+ * @param jobID The job id which the slot allocated for
+ * @param resourceProfile The needed resource profile
+ * @return The future of allocated simple slot
+ */
+ public Future<SimpleSlot> allocateSimpleSlot(final JobID jobID, final ResourceProfile resourceProfile) {
+ return allocateSimpleSlot(jobID, resourceProfile, new AllocationID());
+ }
+
+
+ /**
+ * Try to allocate a simple slot with specified resource profile and specified allocation id. It's mainly
+ * for testing purpose since we need to specify whatever allocation id we want.
+ */
+ @VisibleForTesting
+ Future<SimpleSlot> allocateSimpleSlot(
+ final JobID jobID,
+ final ResourceProfile resourceProfile,
+ final AllocationID allocationID)
+ {
+ final FlinkCompletableFuture<SlotDescriptor> future = new FlinkCompletableFuture<>();
+
+ internalAllocateSlot(jobID, allocationID, resourceProfile, future);
+
+ final SlotOwner owner = this;
+ return future.thenApplyAsync(
+ new ApplyFunction<SlotDescriptor, SimpleSlot>() {
+ @Override
+ public SimpleSlot apply(SlotDescriptor descriptor) {
+ SimpleSlot slot = new SimpleSlot(
+ descriptor.getJobID(), SlotPool.this,
+ descriptor.getTaskManagerLocation(), descriptor.getSlotNumber(),
+ descriptor.getTaskManagerActorGateway());
+ synchronized (lock) {
+ // double validation since we are out of the lock protection after the slot is granted
+ if (registeredResources.contains(descriptor.getTaskManagerLocation().getResourceID())) {
+ LOG.info("Allocation[{}] Allocated simple slot: {} for job {}.", allocationID, slot, jobID);
+ allocatedSlots.add(allocationID, descriptor, slot);
+ }
+ else {
+ throw new RuntimeException("Resource was marked dead asynchronously.");
+ }
+ }
+ return slot;
+ }
+ },
+ executor
+ );
+ }
+
+
+ /**
+ * Try to allocate a shared slot with specified resource profile.
+ *
+ * @param jobID The job id which the slot allocated for
+ * @param resourceProfile The needed resource profile
+ * @param sharingGroupAssignment The slot sharing group of the vertex
+ * @return The future of allocated shared slot
+ */
+ public Future<SharedSlot> allocateSharedSlot(
+ final JobID jobID,
+ final ResourceProfile resourceProfile,
+ final SlotSharingGroupAssignment sharingGroupAssignment)
+ {
+ return allocateSharedSlot(jobID, resourceProfile, sharingGroupAssignment, new AllocationID());
+ }
+
+ /**
+ * Try to allocate a shared slot with specified resource profile and specified allocation id. It's mainly
+ * for testing purpose since we need to specify whatever allocation id we want.
+ */
+ @VisibleForTesting
+ Future<SharedSlot> allocateSharedSlot(
+ final JobID jobID,
+ final ResourceProfile resourceProfile,
+ final SlotSharingGroupAssignment sharingGroupAssignment,
+ final AllocationID allocationID)
+ {
+ final FlinkCompletableFuture<SlotDescriptor> future = new FlinkCompletableFuture<>();
+
+ internalAllocateSlot(jobID, allocationID, resourceProfile, future);
+
+ return future.thenApplyAsync(
+ new ApplyFunction<SlotDescriptor, SharedSlot>() {
+ @Override
+ public SharedSlot apply(SlotDescriptor descriptor) {
+ SharedSlot slot = new SharedSlot(
+ descriptor.getJobID(), SlotPool.this, descriptor.getTaskManagerLocation(),
+ descriptor.getSlotNumber(), descriptor.getTaskManagerActorGateway(),
+ sharingGroupAssignment);
+
+ synchronized (lock) {
+ // double validation since we are out of the lock protection after the slot is granted
+ if (registeredResources.contains(descriptor.getTaskManagerLocation().getResourceID())) {
+ LOG.info("Allocation[{}] Allocated shared slot: {} for job {}.", allocationID, slot, jobID);
+ allocatedSlots.add(allocationID, descriptor, slot);
+ }
+ else {
+ throw new RuntimeException("Resource was marked dead asynchronously.");
+ }
+ }
+ return slot;
+ }
+ },
+ executor
+ );
+ }
+
+ /**
+ * Internally allocate the slot with specified resource profile. We will first check whether we have some
+ * free slot which can meet the requirement already and allocate it immediately. Otherwise, we will try to
+ * allocation the slot from resource manager.
+ */
+ private void internalAllocateSlot(
+ final JobID jobID,
+ final AllocationID allocationID,
+ final ResourceProfile resourceProfile,
+ final FlinkCompletableFuture<SlotDescriptor> future)
+ {
+ LOG.info("Allocation[{}] Allocating slot with {} for Job {}.", allocationID, resourceProfile, jobID);
+
+ synchronized (lock) {
+ // check whether we have any free slot which can match the required resource profile
+ SlotDescriptor freeSlot = availableSlots.poll(resourceProfile);
+ if (freeSlot != null) {
+ future.complete(freeSlot);
+ }
+ else {
+ if (resourceManagerGateway != null) {
+ LOG.info("Allocation[{}] No available slot exists, trying to allocate from resource manager.",
+ allocationID);
+ SlotRequest slotRequest = new SlotRequest(jobID, allocationID, resourceProfile);
+ pendingRequests.put(allocationID, new Tuple2<>(slotRequest, future));
+ resourceManagerGateway.requestSlot(jobManagerLeaderId, resourceManagerLeaderId, slotRequest, timeout)
+ .handleAsync(new BiFunction<RMSlotRequestReply, Throwable, Void>() {
+ @Override
+ public Void apply(RMSlotRequestReply slotRequestReply, Throwable throwable) {
+ if (throwable != null) {
+ future.completeExceptionally(
+ new Exception("Slot allocation from resource manager failed", throwable));
+ } else if (slotRequestReply instanceof RMSlotRequestRejected) {
+ future.completeExceptionally(
+ new Exception("Slot allocation rejected by resource manager"));
+ }
+ return null;
+ }
+ }, executor);
+ }
+ else {
+ LOG.warn("Allocation[{}] Resource manager not available right now.", allocationID);
+ future.completeExceptionally(new Exception("Resource manager not available right now."));
+ }
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Slot De-allocation
+ // ------------------------------------------------------------------------
+
+ /**
+ * Return the slot back to this pool without releasing it. It's mainly called by failed / cancelled tasks, and the
+ * slot can be reused by other pending requests if the resource profile matches.n
+ *
+ * @param slot The slot needs to be returned
+ * @return True if the returning slot been accepted
+ */
+ @Override
+ public boolean returnAllocatedSlot(Slot slot) {
+ checkNotNull(slot);
+ checkArgument(!slot.isAlive(), "slot is still alive");
+ checkArgument(slot.getOwner() == this, "slot belongs to the wrong pool.");
+
+ if (slot.markReleased()) {
+ synchronized (lock) {
+ final SlotDescriptor slotDescriptor = allocatedSlots.remove(slot);
+ if (slotDescriptor != null) {
+ // check if this TaskManager is valid
+ if (!registeredResources.contains(slot.getTaskManagerID())) {
+ return false;
+ }
+
+ final FlinkCompletableFuture<SlotDescriptor> pendingRequest = pollPendingRequest(slotDescriptor);
+ if (pendingRequest != null) {
+ pendingRequest.complete(slotDescriptor);
+ }
+ else {
+ availableSlots.add(slotDescriptor);
+ }
+
+ return true;
+ }
+ else {
+ throw new IllegalArgumentException("Slot was not allocated from this pool.");
+ }
+ }
+ }
+ else {
+ return false;
+ }
+ }
+
+ private FlinkCompletableFuture<SlotDescriptor> pollPendingRequest(final SlotDescriptor slotDescriptor) {
+ for (Map.Entry<AllocationID, Tuple2<SlotRequest, FlinkCompletableFuture<SlotDescriptor>>> entry : pendingRequests.entrySet()) {
+ final Tuple2<SlotRequest, FlinkCompletableFuture<SlotDescriptor>> pendingRequest = entry.getValue();
+ if (slotDescriptor.getResourceProfile().isMatching(pendingRequest.f0.getResourceProfile())) {
+ pendingRequests.remove(entry.getKey());
+ return pendingRequest.f1;
+ }
+ }
+ return null;
+ }
+
+ // ------------------------------------------------------------------------
+ // Slot Releasing
+ // ------------------------------------------------------------------------
+
+ /**
+ * Release slot to TaskManager, called for finished tasks or canceled jobs.
+ *
+ * @param slot The slot needs to be released.
+ */
+ public void releaseSlot(final Slot slot) {
+ synchronized (lock) {
+ allocatedSlots.remove(slot);
+ availableSlots.remove(new SlotDescriptor(slot));
+ // TODO: send release request to task manager
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Slot Offering
+ // ------------------------------------------------------------------------
+
+ /**
+ * Slot offering by TaskManager with AllocationID. The AllocationID is originally generated by this pool and
+ * transfer through the ResourceManager to TaskManager. We use it to distinguish the different allocation
+ * we issued. Slot offering may be rejected if we find something mismatching or there is actually no pending
+ * request waiting for this slot (maybe fulfilled by some other returned slot).
+ *
+ * @param allocationID The allocation id of the lo
+ * @param slotDescriptor The offered slot descriptor
+ * @return True if we accept the offering
+ */
+ public boolean offerSlot(final AllocationID allocationID, final SlotDescriptor slotDescriptor) {
+ synchronized (lock) {
+ // check if this TaskManager is valid
+ final ResourceID resourceID = slotDescriptor.getTaskManagerLocation().getResourceID();
+ if (!registeredResources.contains(resourceID)) {
+ LOG.warn("Allocation[{}] Slot offering from unregistered TaskManager: {}",
+ allocationID, slotDescriptor);
+ return false;
+ }
+
+ // check whether we have already using this slot
+ final Slot allocatedSlot = allocatedSlots.get(allocationID);
+ if (allocatedSlot != null) {
+ final SlotDescriptor allocatedSlotDescriptor = new SlotDescriptor(allocatedSlot);
+
+ if (allocatedSlotDescriptor.equals(slotDescriptor)) {
+ LOG.debug("Allocation[{}] Duplicated slot offering: {}",
+ allocationID, slotDescriptor);
+ return true;
+ }
+ else {
+ LOG.info("Allocation[{}] Allocation had been fulfilled by slot {}, rejecting offered slot {}",
+ allocationID, allocatedSlotDescriptor, slotDescriptor);
+ return false;
+ }
+ }
+
+ // check whether we already have this slot in free pool
+ if (availableSlots.contains(slotDescriptor)) {
+ LOG.debug("Allocation[{}] Duplicated slot offering: {}",
+ allocationID, slotDescriptor);
+ return true;
+ }
+
+ // check whether we have request waiting for this slot
+ if (pendingRequests.containsKey(allocationID)) {
+ FlinkCompletableFuture<SlotDescriptor> future = pendingRequests.remove(allocationID).f1;
+ future.complete(slotDescriptor);
+ return true;
+ }
+
+ // unwanted slot, rejecting this offer
+ return false;
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Resource
+ // ------------------------------------------------------------------------
+
+ /**
+ * Register TaskManager to this pool, only those slots come from registered TaskManager will be considered valid.
+ * Also it provides a way for us to keep "dead" or "abnormal" TaskManagers out of this pool.
+ *
+ * @param resourceID The id of the TaskManager
+ */
+ public void registerResource(final ResourceID resourceID) {
+ synchronized (lock) {
+ registeredResources.add(resourceID);
+ }
+ }
+
+ /**
+ * Unregister TaskManager from this pool, all the related slots will be released and tasks be canceled. Called
+ * when we find some TaskManager becomes "dead" or "abnormal", and we decide to not using slots from it anymore.
+ *
+ * @param resourceID The id of the TaskManager
+ */
+ public void releaseResource(final ResourceID resourceID) {
+ synchronized (lock) {
+ registeredResources.remove(resourceID);
+ availableSlots.removeByResource(resourceID);
+
+ final Set<Slot> allocatedSlotsForResource = allocatedSlots.getSlotsByResource(resourceID);
+ for (Slot slot : allocatedSlotsForResource) {
+ slot.releaseSlot();
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // ResourceManager
+ // ------------------------------------------------------------------------
+
+ public void setResourceManager(
+ final UUID resourceManagerLeaderId,
+ final ResourceManagerGateway resourceManagerGateway)
+ {
+ synchronized (lock) {
+ this.resourceManagerLeaderId = resourceManagerLeaderId;
+ this.resourceManagerGateway = resourceManagerGateway;
+ }
+ }
+
+ public void disconnectResourceManager() {
+ synchronized (lock) {
+ this.resourceManagerLeaderId = null;
+ this.resourceManagerGateway = null;
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Helper classes
+ // ------------------------------------------------------------------------
+
+ /**
+ * Organize allocated slots from different points of view.
+ */
+ static class AllocatedSlots {
+
+ /** All allocated slots organized by TaskManager */
+ private final Map<ResourceID, Set<Slot>> allocatedSlotsByResource;
+
+ /** All allocated slots organized by Slot object */
+ private final Map<Slot, AllocationID> allocatedSlots;
+
+ private final Map<Slot, SlotDescriptor> allocatedSlotsWithDescriptor;
+
+ /** All allocated slots organized by AllocationID */
+ private final Map<AllocationID, Slot> allocatedSlotsById;
+
+ AllocatedSlots() {
+ this.allocatedSlotsByResource = new HashMap<>();
+ this.allocatedSlots = new HashMap<>();
+ this.allocatedSlotsWithDescriptor = new HashMap<>();
+ this.allocatedSlotsById = new HashMap<>();
+ }
+
+ /**
+ * Add a new allocation
+ *
+ * @param allocationID The allocation id
+ * @param slot The allocated slot
+ */
+ void add(final AllocationID allocationID, final SlotDescriptor descriptor, final Slot slot) {
+ allocatedSlots.put(slot, allocationID);
+ allocatedSlotsById.put(allocationID, slot);
+ allocatedSlotsWithDescriptor.put(slot, descriptor);
+
+ final ResourceID resourceID = slot.getTaskManagerID();
+ Set<Slot> slotsForResource = allocatedSlotsByResource.get(resourceID);
+ if (slotsForResource == null) {
+ slotsForResource = new HashSet<>();
+ allocatedSlotsByResource.put(resourceID, slotsForResource);
+ }
+ slotsForResource.add(slot);
+ }
+
+ /**
+ * Get allocated slot with allocation id
+ *
+ * @param allocationID The allocation id
+ * @return The allocated slot, null if we can't find a match
+ */
+ Slot get(final AllocationID allocationID) {
+ return allocatedSlotsById.get(allocationID);
+ }
+
+ /**
+ * Check whether we have allocated this slot
+ *
+ * @param slot The slot needs to checked
+ * @return True if we contains this slot
+ */
+ boolean contains(final Slot slot) {
+ return allocatedSlots.containsKey(slot);
+ }
+
+ /**
+ * Remove an allocation with slot.
+ *
+ * @param slot The slot needs to be removed
+ */
+ SlotDescriptor remove(final Slot slot) {
+ final SlotDescriptor descriptor = allocatedSlotsWithDescriptor.remove(slot);
+ if (descriptor != null) {
+ final AllocationID allocationID = allocatedSlots.remove(slot);
+ if (allocationID != null) {
+ allocatedSlotsById.remove(allocationID);
+ } else {
+ throw new IllegalStateException("Bug: maps are inconsistent");
+ }
+
+ final ResourceID resourceID = slot.getTaskManagerID();
+ final Set<Slot> slotsForResource = allocatedSlotsByResource.get(resourceID);
+ slotsForResource.remove(slot);
+ if (slotsForResource.isEmpty()) {
+ allocatedSlotsByResource.remove(resourceID);
+ }
+
+ return descriptor;
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Get all allocated slot from same TaskManager.
+ *
+ * @param resourceID The id of the TaskManager
+ * @return Set of slots which are allocated from the same TaskManager
+ */
+ Set<Slot> getSlotsByResource(final ResourceID resourceID) {
+ Set<Slot> slotsForResource = allocatedSlotsByResource.get(resourceID);
+ if (slotsForResource != null) {
+ return new HashSet<>(slotsForResource);
+ }
+ else {
+ return new HashSet<>();
+ }
+ }
+
+ @VisibleForTesting
+ boolean containResource(final ResourceID resourceID) {
+ return allocatedSlotsByResource.containsKey(resourceID);
+ }
+
+ @VisibleForTesting
+ int size() {
+ return allocatedSlots.size();
+ }
+ }
+
+ /**
+ * Organize all available slots from different points of view.
+ */
+ static class AvailableSlots {
+
+ /** All available slots organized by TaskManager */
+ private final Map<ResourceID, Set<SlotDescriptor>> availableSlotsByResource;
+
+ /** All available slots */
+ private final Set<SlotDescriptor> availableSlots;
+
+ AvailableSlots() {
+ this.availableSlotsByResource = new HashMap<>();
+ this.availableSlots = new HashSet<>();
+ }
+
+ /**
+ * Add an available slot.
+ *
+ * @param descriptor The descriptor of the slot
+ */
+ void add(final SlotDescriptor descriptor) {
+ availableSlots.add(descriptor);
+
+ final ResourceID resourceID = descriptor.getTaskManagerLocation().getResourceID();
+ Set<SlotDescriptor> slotsForResource = availableSlotsByResource.get(resourceID);
+ if (slotsForResource == null) {
+ slotsForResource = new HashSet<>();
+ availableSlotsByResource.put(resourceID, slotsForResource);
+ }
+ slotsForResource.add(descriptor);
+ }
+
+ /**
+ * Check whether we have this slot
+ *
+ * @param slotDescriptor The descriptor of the slot
+ * @return True if we contains this slot
+ */
+ boolean contains(final SlotDescriptor slotDescriptor) {
+ return availableSlots.contains(slotDescriptor);
+ }
+
+ /**
+ * Poll a slot which matches the required resource profile
+ *
+ * @param resourceProfile The required resource profile
+ * @return Slot which matches the resource profile, null if we can't find a match
+ */
+ SlotDescriptor poll(final ResourceProfile resourceProfile) {
+ for (SlotDescriptor slotDescriptor : availableSlots) {
+ if (slotDescriptor.getResourceProfile().isMatching(resourceProfile)) {
+ remove(slotDescriptor);
+ return slotDescriptor;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Remove all available slots come from specified TaskManager.
+ *
+ * @param resourceID The id of the TaskManager
+ */
+ void removeByResource(final ResourceID resourceID) {
+ final Set<SlotDescriptor> slotsForResource = availableSlotsByResource.remove(resourceID);
+ if (slotsForResource != null) {
+ for (SlotDescriptor slotDescriptor : slotsForResource) {
+ availableSlots.remove(slotDescriptor);
+ }
+ }
+ }
+
+ private void remove(final SlotDescriptor slotDescriptor) {
+ availableSlots.remove(slotDescriptor);
+
+ final ResourceID resourceID = slotDescriptor.getTaskManagerLocation().getResourceID();
+ final Set<SlotDescriptor> slotsForResource = checkNotNull(availableSlotsByResource.get(resourceID));
+ slotsForResource.remove(slotDescriptor);
+ if (slotsForResource.isEmpty()) {
+ availableSlotsByResource.remove(resourceID);
+ }
+ }
+
+ @VisibleForTesting
+ boolean containResource(final ResourceID resourceID) {
+ return availableSlotsByResource.containsKey(resourceID);
+ }
+
+ @VisibleForTesting
+ int size() {
+ return availableSlots.size();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1801de9c/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
index 8394902..1f23779 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala
@@ -660,8 +660,8 @@ object AkkaUtils {
}
def formatDurationParingErrorMessage: String = {
- "Duration format must be \"val unit\", where 'val' is a number and 'unit' is " +
- "(d|day)|(h|hour)|(min|minute)|s|sec|second)|(ms|milli|millisecond)|"+
+ "Duration format must be \"val unit\", where 'val' is a number and 'unit' is " +
+ "(d|day)|(h|hour)|(min|minute)|s|sec|second)|(ms|milli|millisecond)|" +
"(�s|micro|microsecond)|(ns|nano|nanosecond)"
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1801de9c/flink-runtime/src/test/java/org/apache/flink/runtime/instance/AllocatedSlotsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/AllocatedSlotsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/AllocatedSlotsTest.java
new file mode 100644
index 0000000..655a3ea
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/AllocatedSlotsTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.instance;
+
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class AllocatedSlotsTest {
+
+ @Test
+ public void testOperations() throws Exception {
+ SlotPool.AllocatedSlots allocatedSlots = new SlotPool.AllocatedSlots();
+
+ final AllocationID allocation1 = new AllocationID();
+ final ResourceID resource1 = new ResourceID("resource1");
+ final Slot slot1 = createSlot(resource1);
+
+ allocatedSlots.add(allocation1, new SlotDescriptor(slot1), slot1);
+
+ assertTrue(allocatedSlots.contains(slot1));
+ assertTrue(allocatedSlots.containResource(resource1));
+
+ assertEquals(slot1, allocatedSlots.get(allocation1));
+ assertEquals(1, allocatedSlots.getSlotsByResource(resource1).size());
+ assertEquals(1, allocatedSlots.size());
+
+ final AllocationID allocation2 = new AllocationID();
+ final Slot slot2 = createSlot(resource1);
+
+ allocatedSlots.add(allocation2, new SlotDescriptor(slot2), slot2);
+
+ assertTrue(allocatedSlots.contains(slot1));
+ assertTrue(allocatedSlots.contains(slot2));
+ assertTrue(allocatedSlots.containResource(resource1));
+
+ assertEquals(slot1, allocatedSlots.get(allocation1));
+ assertEquals(slot2, allocatedSlots.get(allocation2));
+ assertEquals(2, allocatedSlots.getSlotsByResource(resource1).size());
+ assertEquals(2, allocatedSlots.size());
+
+ final AllocationID allocation3 = new AllocationID();
+ final ResourceID resource2 = new ResourceID("resource2");
+ final Slot slot3 = createSlot(resource2);
+
+ allocatedSlots.add(allocation3, new SlotDescriptor(slot2), slot3);
+
+ assertTrue(allocatedSlots.contains(slot1));
+ assertTrue(allocatedSlots.contains(slot2));
+ assertTrue(allocatedSlots.contains(slot3));
+ assertTrue(allocatedSlots.containResource(resource1));
+ assertTrue(allocatedSlots.containResource(resource2));
+
+ assertEquals(slot1, allocatedSlots.get(allocation1));
+ assertEquals(slot2, allocatedSlots.get(allocation2));
+ assertEquals(slot3, allocatedSlots.get(allocation3));
+ assertEquals(2, allocatedSlots.getSlotsByResource(resource1).size());
+ assertEquals(1, allocatedSlots.getSlotsByResource(resource2).size());
+ assertEquals(3, allocatedSlots.size());
+
+ allocatedSlots.remove(slot2);
+
+ assertTrue(allocatedSlots.contains(slot1));
+ assertFalse(allocatedSlots.contains(slot2));
+ assertTrue(allocatedSlots.contains(slot3));
+ assertTrue(allocatedSlots.containResource(resource1));
+ assertTrue(allocatedSlots.containResource(resource2));
+
+ assertEquals(slot1, allocatedSlots.get(allocation1));
+ assertNull(allocatedSlots.get(allocation2));
+ assertEquals(slot3, allocatedSlots.get(allocation3));
+ assertEquals(1, allocatedSlots.getSlotsByResource(resource1).size());
+ assertEquals(1, allocatedSlots.getSlotsByResource(resource2).size());
+ assertEquals(2, allocatedSlots.size());
+
+ allocatedSlots.remove(slot1);
+
+ assertFalse(allocatedSlots.contains(slot1));
+ assertFalse(allocatedSlots.contains(slot2));
+ assertTrue(allocatedSlots.contains(slot3));
+ assertFalse(allocatedSlots.containResource(resource1));
+ assertTrue(allocatedSlots.containResource(resource2));
+
+ assertNull(allocatedSlots.get(allocation1));
+ assertNull(allocatedSlots.get(allocation2));
+ assertEquals(slot3, allocatedSlots.get(allocation3));
+ assertEquals(0, allocatedSlots.getSlotsByResource(resource1).size());
+ assertEquals(1, allocatedSlots.getSlotsByResource(resource2).size());
+ assertEquals(1, allocatedSlots.size());
+
+ allocatedSlots.remove(slot3);
+
+ assertFalse(allocatedSlots.contains(slot1));
+ assertFalse(allocatedSlots.contains(slot2));
+ assertFalse(allocatedSlots.contains(slot3));
+ assertFalse(allocatedSlots.containResource(resource1));
+ assertFalse(allocatedSlots.containResource(resource2));
+
+ assertNull(allocatedSlots.get(allocation1));
+ assertNull(allocatedSlots.get(allocation2));
+ assertNull(allocatedSlots.get(allocation3));
+ assertEquals(0, allocatedSlots.getSlotsByResource(resource1).size());
+ assertEquals(0, allocatedSlots.getSlotsByResource(resource2).size());
+ assertEquals(0, allocatedSlots.size());
+ }
+
+ private Slot createSlot(final ResourceID resourceId) {
+ Slot slot = mock(Slot.class);
+ when(slot.getTaskManagerID()).thenReturn(resourceId);
+ return slot;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1801de9c/flink-runtime/src/test/java/org/apache/flink/runtime/instance/AvailableSlotsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/AvailableSlotsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/AvailableSlotsTest.java
new file mode 100644
index 0000000..872810f
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/AvailableSlotsTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.instance;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
+import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class AvailableSlotsTest {
+
+ static final ResourceProfile DEFAULT_TESTING_PROFILE = new ResourceProfile(1.0, 512);
+
+ static final ResourceProfile DEFAULT_TESTING_BIG_PROFILE = new ResourceProfile(2.0, 1024);
+
+ @Test
+ public void testAddAndRemove() throws Exception {
+ SlotPool.AvailableSlots availableSlots = new SlotPool.AvailableSlots();
+
+ final ResourceID resource1 = new ResourceID("resource1");
+ final ResourceID resource2 = new ResourceID("resource2");
+
+ final SlotDescriptor slot1 = createSlotDescriptor(resource1);
+ final SlotDescriptor slot2 = createSlotDescriptor(resource1);
+ final SlotDescriptor slot3 = createSlotDescriptor(resource2);
+
+ availableSlots.add(slot1);
+ availableSlots.add(slot2);
+ availableSlots.add(slot3);
+
+ assertEquals(3, availableSlots.size());
+ assertTrue(availableSlots.contains(slot1));
+ assertTrue(availableSlots.contains(slot2));
+ assertTrue(availableSlots.contains(slot3));
+ assertTrue(availableSlots.containResource(resource1));
+ assertTrue(availableSlots.containResource(resource2));
+
+ availableSlots.removeByResource(resource1);
+
+ assertEquals(1, availableSlots.size());
+ assertFalse(availableSlots.contains(slot1));
+ assertFalse(availableSlots.contains(slot2));
+ assertTrue(availableSlots.contains(slot3));
+ assertFalse(availableSlots.containResource(resource1));
+ assertTrue(availableSlots.containResource(resource2));
+
+ availableSlots.removeByResource(resource2);
+
+ assertEquals(0, availableSlots.size());
+ assertFalse(availableSlots.contains(slot1));
+ assertFalse(availableSlots.contains(slot2));
+ assertFalse(availableSlots.contains(slot3));
+ assertFalse(availableSlots.containResource(resource1));
+ assertFalse(availableSlots.containResource(resource2));
+ }
+
+ @Test
+ public void testPollFreeSlot() {
+ SlotPool.AvailableSlots availableSlots = new SlotPool.AvailableSlots();
+
+ final ResourceID resource1 = new ResourceID("resource1");
+ final SlotDescriptor slot1 = createSlotDescriptor(resource1);
+
+ availableSlots.add(slot1);
+
+ assertEquals(1, availableSlots.size());
+ assertTrue(availableSlots.contains(slot1));
+ assertTrue(availableSlots.containResource(resource1));
+
+ assertNull(availableSlots.poll(DEFAULT_TESTING_BIG_PROFILE));
+
+ assertEquals(slot1, availableSlots.poll(DEFAULT_TESTING_PROFILE));
+ assertEquals(0, availableSlots.size());
+ assertFalse(availableSlots.contains(slot1));
+ assertFalse(availableSlots.containResource(resource1));
+ }
+
+ static SlotDescriptor createSlotDescriptor(final ResourceID resourceID) {
+ return createSlotDescriptor(resourceID, new JobID());
+ }
+
+ static SlotDescriptor createSlotDescriptor(final ResourceID resourceID, final JobID jobID) {
+ return createSlotDescriptor(resourceID, jobID, DEFAULT_TESTING_PROFILE);
+ }
+
+ static SlotDescriptor createSlotDescriptor(final ResourceID resourceID, final JobID jobID,
+ final ResourceProfile resourceProfile)
+ {
+ return createSlotDescriptor(resourceID, jobID, resourceProfile, 0);
+ }
+
+ static SlotDescriptor createSlotDescriptor(final ResourceID resourceID, final JobID jobID,
+ final ResourceProfile resourceProfile, final int slotNumber)
+ {
+ TaskManagerLocation location = mock(TaskManagerLocation.class);
+ when(location.getResourceID()).thenReturn(resourceID);
+ return new SlotDescriptor(jobID, location, slotNumber, resourceProfile, mock(ActorGateway.class));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1801de9c/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotPoolTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotPoolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotPoolTest.java
new file mode 100644
index 0000000..30cdbd6
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotPoolTest.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.instance;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.clusterframework.types.AllocationID;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.BiFunction;
+import org.apache.flink.runtime.concurrent.Future;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
+import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway;
+import org.apache.flink.runtime.resourcemanager.SlotRequest;
+import org.apache.flink.util.TestLogger;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.runtime.instance.AvailableSlotsTest.DEFAULT_TESTING_PROFILE;
+import static org.apache.flink.runtime.instance.AvailableSlotsTest.createSlotDescriptor;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class SlotPoolTest extends TestLogger {
+
+ private Executor executor;
+
+ private SlotPool slotPool;
+
+ private ResourceManagerGateway resourceManagerGateway;
+
+ @Before
+ public void setUp() throws Exception {
+ this.executor = Executors.newFixedThreadPool(1);
+ this.slotPool = new SlotPool(executor);
+ this.resourceManagerGateway = mock(ResourceManagerGateway.class);
+ when(resourceManagerGateway
+ .requestSlot(any(UUID.class), any(UUID.class), any(SlotRequest.class), any(Time.class)))
+ .thenReturn(mock(Future.class));
+ slotPool.setResourceManager(UUID.randomUUID(), resourceManagerGateway);
+ slotPool.setJobManagerLeaderId(UUID.randomUUID());
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ @Test
+ public void testAllocateSimpleSlot() throws Exception {
+ ResourceID resourceID = new ResourceID("resource");
+ slotPool.registerResource(resourceID);
+
+ JobID jobID = new JobID();
+ AllocationID allocationID = new AllocationID();
+ Future<SimpleSlot> future = slotPool.allocateSimpleSlot(jobID, DEFAULT_TESTING_PROFILE, allocationID);
+ assertFalse(future.isDone());
+ verify(resourceManagerGateway).requestSlot(any(UUID.class), any(UUID.class), any(SlotRequest.class), any(Time.class));
+
+ SlotDescriptor slotDescriptor = createSlotDescriptor(resourceID, jobID, DEFAULT_TESTING_PROFILE);
+ assertTrue(slotPool.offerSlot(allocationID, slotDescriptor));
+
+ SimpleSlot slot = future.get(1, TimeUnit.SECONDS);
+ assertTrue(future.isDone());
+ assertTrue(slot.isAlive());
+ assertEquals(resourceID, slot.getTaskManagerID());
+ assertEquals(jobID, slot.getJobID());
+ assertEquals(slotPool, slot.getOwner());
+ }
+
+ @Test
+ public void testAllocateSharedSlot() throws Exception {
+ ResourceID resourceID = new ResourceID("resource");
+ slotPool.registerResource(resourceID);
+
+ JobVertexID vid = new JobVertexID();
+ SlotSharingGroup sharingGroup = new SlotSharingGroup(vid);
+ SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();
+
+ JobID jobID = new JobID();
+ AllocationID allocationID = new AllocationID();
+ Future<SharedSlot> future = slotPool.allocateSharedSlot(jobID, DEFAULT_TESTING_PROFILE, assignment, allocationID);
+
+ assertFalse(future.isDone());
+ verify(resourceManagerGateway).requestSlot(any(UUID.class), any(UUID.class), any(SlotRequest.class), any(Time.class));
+
+ SlotDescriptor slotDescriptor = createSlotDescriptor(resourceID, jobID, DEFAULT_TESTING_PROFILE);
+ assertTrue(slotPool.offerSlot(allocationID, slotDescriptor));
+
+ SharedSlot slot = future.get(1, TimeUnit.SECONDS);
+ assertTrue(future.isDone());
+ assertTrue(slot.isAlive());
+ assertEquals(resourceID, slot.getTaskManagerID());
+ assertEquals(jobID, slot.getJobID());
+ assertEquals(slotPool, slot.getOwner());
+
+ SimpleSlot simpleSlot = slot.allocateSubSlot(vid);
+ assertNotNull(simpleSlot);
+ assertTrue(simpleSlot.isAlive());
+ }
+
+ @Test
+ public void testAllocateSlotWithoutResourceManager() throws Exception {
+ slotPool.disconnectResourceManager();
+ Future<SimpleSlot> future = slotPool.allocateSimpleSlot(new JobID(), DEFAULT_TESTING_PROFILE);
+ future.handleAsync(
+ new BiFunction<SimpleSlot, Throwable, Void>() {
+ @Override
+ public Void apply(SimpleSlot simpleSlot, Throwable throwable) {
+ assertNull(simpleSlot);
+ assertNotNull(throwable);
+ return null;
+ }
+ },
+ executor);
+ try {
+ future.get(1, TimeUnit.SECONDS);
+ fail("We expected a ExecutionException.");
+ } catch (ExecutionException ex) {
+ // we expect the exception
+ }
+ }
+
+ @Test
+ public void testAllocationFulfilledByReturnedSlot() throws Exception {
+ ResourceID resourceID = new ResourceID("resource");
+ slotPool.registerResource(resourceID);
+
+ JobID jobID = new JobID();
+
+ AllocationID allocationID1 = new AllocationID();
+ Future<SimpleSlot> future1 = slotPool.allocateSimpleSlot(jobID, DEFAULT_TESTING_PROFILE, allocationID1);
+
+ AllocationID allocationID2 = new AllocationID();
+ Future<SimpleSlot> future2 = slotPool.allocateSimpleSlot(jobID, DEFAULT_TESTING_PROFILE, allocationID2);
+
+ assertFalse(future1.isDone());
+ assertFalse(future2.isDone());
+ verify(resourceManagerGateway, times(2))
+ .requestSlot(any(UUID.class), any(UUID.class), any(SlotRequest.class), any(Time.class));
+
+ SlotDescriptor slotDescriptor = createSlotDescriptor(resourceID, jobID, DEFAULT_TESTING_PROFILE);
+ assertTrue(slotPool.offerSlot(allocationID1, slotDescriptor));
+
+ SimpleSlot slot1 = future1.get(1, TimeUnit.SECONDS);
+ assertTrue(future1.isDone());
+ assertFalse(future2.isDone());
+
+ // return this slot to pool
+ slot1.releaseSlot();
+
+ // second allocation fulfilled by previous slot returning
+ SimpleSlot slot2 = future2.get(1, TimeUnit.SECONDS);
+ assertTrue(future2.isDone());
+
+ assertNotEquals(slot1, slot2);
+ assertTrue(slot1.isReleased());
+ assertTrue(slot2.isAlive());
+ assertEquals(slot1.getTaskManagerID(), slot2.getTaskManagerID());
+ assertEquals(slot1.getSlotNumber(), slot2.getSlotNumber());
+ }
+
+ @Test
+ public void testAllocateWithFreeSlot() throws Exception {
+ ResourceID resourceID = new ResourceID("resource");
+ slotPool.registerResource(resourceID);
+
+ JobID jobID = new JobID();
+ AllocationID allocationID1 = new AllocationID();
+ Future<SimpleSlot> future1 = slotPool.allocateSimpleSlot(jobID, DEFAULT_TESTING_PROFILE, allocationID1);
+ assertFalse(future1.isDone());
+
+ SlotDescriptor slotDescriptor = createSlotDescriptor(resourceID, jobID, DEFAULT_TESTING_PROFILE);
+ assertTrue(slotPool.offerSlot(allocationID1, slotDescriptor));
+
+ SimpleSlot slot1 = future1.get(1, TimeUnit.SECONDS);
+ assertTrue(future1.isDone());
+
+ // return this slot to pool
+ slot1.releaseSlot();
+
+ AllocationID allocationID2 = new AllocationID();
+ Future<SimpleSlot> future2 = slotPool.allocateSimpleSlot(jobID, DEFAULT_TESTING_PROFILE, allocationID2);
+
+ // second allocation fulfilled by previous slot returning
+ SimpleSlot slot2 = future2.get(1, TimeUnit.SECONDS);
+ assertTrue(future2.isDone());
+
+ assertNotEquals(slot1, slot2);
+ assertTrue(slot1.isReleased());
+ assertTrue(slot2.isAlive());
+ assertEquals(slot1.getTaskManagerID(), slot2.getTaskManagerID());
+ assertEquals(slot1.getSlotNumber(), slot2.getSlotNumber());
+ }
+
+ @Test
+ public void testOfferSlot() throws Exception {
+ ResourceID resourceID = new ResourceID("resource");
+ slotPool.registerResource(resourceID);
+
+ JobID jobID = new JobID();
+ AllocationID allocationID = new AllocationID();
+ Future<SimpleSlot> future = slotPool.allocateSimpleSlot(jobID, DEFAULT_TESTING_PROFILE, allocationID);
+ assertFalse(future.isDone());
+ verify(resourceManagerGateway).requestSlot(any(UUID.class), any(UUID.class), any(SlotRequest.class), any(Time.class));
+
+ // slot from unregistered resource
+ SlotDescriptor invalid = createSlotDescriptor(new ResourceID("unregistered"), jobID, DEFAULT_TESTING_PROFILE);
+ assertFalse(slotPool.offerSlot(allocationID, invalid));
+
+ SlotDescriptor slotDescriptor = createSlotDescriptor(resourceID, jobID, DEFAULT_TESTING_PROFILE);
+
+ // reject offering with mismatch allocation id
+ assertFalse(slotPool.offerSlot(new AllocationID(), slotDescriptor));
+
+ // accepted slot
+ assertTrue(slotPool.offerSlot(allocationID, slotDescriptor));
+ SimpleSlot slot = future.get(1, TimeUnit.SECONDS);
+ assertTrue(future.isDone());
+ assertTrue(slot.isAlive());
+
+ // conflict offer with using slot
+ SlotDescriptor conflict = createSlotDescriptor(resourceID, jobID, DEFAULT_TESTING_PROFILE);
+ assertFalse(slotPool.offerSlot(allocationID, conflict));
+
+ // duplicated offer with using slot
+ assertTrue(slotPool.offerSlot(allocationID, slotDescriptor));
+ assertTrue(future.isDone());
+ assertTrue(slot.isAlive());
+
+ // duplicated offer with free slot
+ slot.releaseSlot();
+ assertTrue(slot.isReleased());
+ assertTrue(slotPool.offerSlot(allocationID, slotDescriptor));
+ }
+
+ @Test
+ public void testReleaseResource() throws Exception {
+ ResourceID resourceID = new ResourceID("resource");
+ slotPool.registerResource(resourceID);
+
+ JobID jobID = new JobID();
+
+ AllocationID allocationID1 = new AllocationID();
+ Future<SimpleSlot> future1 = slotPool.allocateSimpleSlot(jobID, DEFAULT_TESTING_PROFILE, allocationID1);
+
+ AllocationID allocationID2 = new AllocationID();
+ Future<SimpleSlot> future2 = slotPool.allocateSimpleSlot(jobID, DEFAULT_TESTING_PROFILE, allocationID2);
+
+ SlotDescriptor slotDescriptor = createSlotDescriptor(resourceID, jobID, DEFAULT_TESTING_PROFILE);
+ assertTrue(slotPool.offerSlot(allocationID1, slotDescriptor));
+
+ SimpleSlot slot1 = future1.get(1, TimeUnit.SECONDS);
+ assertTrue(future1.isDone());
+ assertFalse(future2.isDone());
+
+ slotPool.releaseResource(resourceID);
+ assertTrue(slot1.isReleased());
+
+ // slot released and not usable, second allocation still not fulfilled
+ Thread.sleep(10);
+ assertFalse(future2.isDone());
+ }
+
+}