You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2015/02/17 13:17:23 UTC
[1/3] flink git commit: [FLINK-1549] [yarn] Adds proper exception
handling to YarnJobManager
Repository: flink
Updated Branches:
refs/heads/master c1e326707 -> ac7bf4279
[FLINK-1549] [yarn] Adds proper exception handling to YarnJobManager
This closes #397.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/447ce0aa
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/447ce0aa
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/447ce0aa
Branch: refs/heads/master
Commit: 447ce0aa60358212a9716f6b4319f7e9566d6eaa
Parents: c1e3267
Author: Till Rohrmann <tr...@apache.org>
Authored: Sat Feb 14 23:36:45 2015 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Feb 17 11:53:34 2015 +0100
----------------------------------------------------------------------
.../flink/client/FlinkYarnSessionCli.java | 6 +-
.../flink/runtime/jobmanager/JobManager.scala | 2 +-
.../org/apache/flink/yarn/FlinkYarnCluster.java | 2 +-
.../apache/flink/yarn/ApplicationClient.scala | 19 ++-
.../scala/org/apache/flink/yarn/Messages.scala | 2 +-
.../org/apache/flink/yarn/YarnJobManager.scala | 152 +++++++++++--------
6 files changed, 108 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/447ce0aa/flink-clients/src/main/java/org/apache/flink/client/FlinkYarnSessionCli.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/FlinkYarnSessionCli.java b/flink-clients/src/main/java/org/apache/flink/client/FlinkYarnSessionCli.java
index 8117ec3..8d5a3c5 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/FlinkYarnSessionCli.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/FlinkYarnSessionCli.java
@@ -407,8 +407,10 @@ public class FlinkYarnSessionCli {
runInteractiveCli(yarnCluster);
- LOG.info("Command Line Interface requested session shutdown");
- yarnCluster.shutdown();
+ if(!yarnCluster.hasBeenStopped()) {
+ LOG.info("Command Line Interface requested session shutdown");
+ yarnCluster.shutdown();
+ }
try {
yarnPropertiesFile.delete();
http://git-wip-us.apache.org/repos/asf/flink/blob/447ce0aa/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index 4fe0ea6..b490b32 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -115,7 +115,7 @@ class JobManager(val configuration: Configuration,
log.info(s"Stopping job manager ${self.path}.")
archive ! PoisonPill
- profiler.map( ref => ref ! PoisonPill )
+ profiler.foreach( ref => ref ! PoisonPill )
for((e,_) <- currentJobs.values){
e.fail(new Exception("The JobManager is shutting down."))
http://git-wip-us.apache.org/repos/asf/flink/blob/447ce0aa/flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnCluster.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnCluster.java b/flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnCluster.java
index 6fedd9a..2f652a1 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnCluster.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnCluster.java
@@ -103,7 +103,7 @@ public class FlinkYarnCluster extends AbstractFlinkYarnCluster {
// start application client
LOG.info("Start application client.");
- applicationClient = actorSystem.actorOf(Props.create(ApplicationClient.class));
+ applicationClient = actorSystem.actorOf(Props.create(ApplicationClient.class), "applicationClient");
// instruct ApplicationClient to start a periodical status polling
applicationClient.tell(new Messages.LocalRegisterClient(this.jobManagerAddress), applicationClient);
http://git-wip-us.apache.org/repos/asf/flink/blob/447ce0aa/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationClient.scala
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationClient.scala b/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationClient.scala
index 684990b..dba6344 100644
--- a/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationClient.scala
+++ b/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationClient.scala
@@ -21,6 +21,7 @@ package org.apache.flink.yarn
import java.net.InetSocketAddress
import akka.actor._
+import akka.pattern.ask
import org.apache.flink.configuration.GlobalConfiguration
import org.apache.flink.runtime.ActorLogMessages
import org.apache.flink.runtime.akka.AkkaUtils
@@ -61,6 +62,9 @@ class ApplicationClient extends Actor with ActorLogMessages with ActorLogging {
}
pollingTimer = None
+
+ // Terminate the whole actor system because there is only the application client running
+ context.system.shutdown()
}
override def receiveWithLogMessages: Receive = {
@@ -75,6 +79,8 @@ class ApplicationClient extends Actor with ActorLogMessages with ActorLogging {
case Failure(t) =>
log.error(t, "Registration at JobManager/ApplicationMaster failed. Shutting " +
"ApplicationClient down.")
+
+ // we could not connect to the job manager --> poison ourselves
self ! PoisonPill
}
@@ -84,7 +90,11 @@ class ApplicationClient extends Actor with ActorLogMessages with ActorLogging {
// the message came from the FlinkYarnCluster. We send the message to the JobManager.
// it is important not to forward the message because the JobManager is storing the
// sender as the Application Client (this class).
- jm ! RegisterClient
+ (jm ? RegisterClient(self))(timeout).onFailure{
+ case t: Throwable =>
+ log.error(t, "Could not register at the job manager.")
+ self ! PoisonPill
+ }
// schedule a periodic status report from the JobManager
// request the number of task managers and slots from the job manager
@@ -93,7 +103,7 @@ class ApplicationClient extends Actor with ActorLogMessages with ActorLogging {
case msg: StopYarnSession =>
log.info("Stop yarn session.")
- stopMessageReceiver = Some(sender())
+ stopMessageReceiver = Some(sender)
yarnJobManager foreach {
_ forward msg
}
@@ -104,9 +114,8 @@ class ApplicationClient extends Actor with ActorLogMessages with ActorLogging {
stopMessageReceiver foreach {
_ ! JobManagerStopped
}
- // stop ourselves
- context.system.shutdown()
-
+ // poison ourselves
+ self ! PoisonPill
// handle the responses from the PollYarnClusterStatus messages to the yarn job mgr
case status: FlinkYarnClusterStatus =>
http://git-wip-us.apache.org/repos/asf/flink/blob/447ce0aa/flink-yarn/src/main/scala/org/apache/flink/yarn/Messages.scala
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/scala/org/apache/flink/yarn/Messages.scala b/flink-yarn/src/main/scala/org/apache/flink/yarn/Messages.scala
index e880fdf..4ae1a3a 100644
--- a/flink-yarn/src/main/scala/org/apache/flink/yarn/Messages.scala
+++ b/flink-yarn/src/main/scala/org/apache/flink/yarn/Messages.scala
@@ -29,7 +29,7 @@ object Messages {
case class YarnMessage(message: String, date: Date = new Date())
case class ApplicationMasterStatus(numTaskManagers: Int, numSlots: Int)
- case object RegisterClient
+ case class RegisterClient(client: ActorRef)
case class StopYarnSession(status: FinalApplicationStatus)
http://git-wip-us.apache.org/repos/asf/flink/blob/447ce0aa/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnJobManager.scala
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnJobManager.scala b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnJobManager.scala
index a37c8b4..81bdc3b 100644
--- a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnJobManager.scala
+++ b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnJobManager.scala
@@ -26,6 +26,7 @@ import akka.actor.ActorRef
import org.apache.flink.configuration.ConfigConstants
import org.apache.flink.runtime.ActorLogMessages
import org.apache.flink.runtime.jobmanager.JobManager
+import org.apache.flink.runtime.messages.Messages.Acknowledge
import org.apache.flink.runtime.yarn.FlinkYarnClusterStatus
import org.apache.flink.yarn.Messages._
import org.apache.flink.yarn.appMaster.YarnTaskManagerRunner
@@ -42,6 +43,7 @@ import org.apache.hadoop.yarn.util.Records
import scala.concurrent.duration._
import scala.language.postfixOps
+import scala.util.Try
trait YarnJobManager extends ActorLogMessages {
@@ -69,7 +71,7 @@ trait YarnJobManager extends ActorLogMessages {
def receiveYarnMessages: Receive = {
case StopYarnSession(status) =>
- log.info("Stopping YARN Session.")
+ log.info("Stopping YARN JobManager with status {}.", status)
instanceManager.getAllRegisteredInstances.asScala foreach {
instance =>
@@ -78,30 +80,96 @@ trait YarnJobManager extends ActorLogMessages {
rmClientOption foreach {
rmClient =>
- rmClient.unregisterApplicationMaster(status, "", "")
- rmClient.close()
+ Try(rmClient.unregisterApplicationMaster(status, "", "")).recover{
+ case t: Throwable => log.error(t, "Could not unregister the application master.")
+ }
+
+ Try(rmClient.close()).recover{
+ case t:Throwable => log.error(t, "Could not close the AMRMClient.")
+ }
}
rmClientOption = None
nmClientOption foreach {
- _.close()
+ nmClient =>
+ Try(nmClient.close()).recover{
+ case t: Throwable => log.error(t, "Could not close the NMClient.")
+ }
}
nmClientOption = None
messageListener foreach {
- _ ! JobManagerStopped
+ _ ! JobManagerStopped
}
+
context.system.shutdown()
- case RegisterClient =>
- messageListener = Some(sender())
+ case RegisterClient(client) =>
+ log.info("Register {} as client.", client.path)
+ messageListener = Some(client)
+ sender ! Acknowledge
case PollYarnClusterStatus =>
sender() ! new FlinkYarnClusterStatus(instanceManager.getNumberOfRegisteredTaskManagers,
instanceManager.getTotalNumberOfSlots)
- case StartYarnSession(conf, actorSystemPort, webServerport) =>
+ case StartYarnSession(conf, actorSystemPort, webServerPort) => startYarnSession(conf, actorSystemPort, webServerPort)
+
+ case PollContainerCompletion =>
+ rmClientOption match {
+ case Some(rmClient) =>
+ val response = rmClient.allocate(completedContainers.toFloat / numTaskManager)
+
+ for (container <- response.getAllocatedContainers.asScala) {
+ log.info(s"Got new container for TM ${container.getId} on host ${
+ container.getNodeId.getHost
+ }")
+
+ allocatedContainers += 1
+
+ log.info(s"Launching container #$allocatedContainers.")
+ nmClientOption match {
+ case Some(nmClient) =>
+ containerLaunchContext match {
+ case Some(ctx) => nmClient.startContainer(container, ctx)
+ case None =>
+ log.error("The ContainerLaunchContext was not set.")
+ self ! StopYarnSession(FinalApplicationStatus.FAILED)
+ }
+ case None =>
+ log.error("The NMClient was not set.")
+ self ! StopYarnSession(FinalApplicationStatus.FAILED)
+ }
+ }
+
+ for (status <- response.getCompletedContainersStatuses.asScala) {
+ completedContainers += 1
+ log.info(s"Completed container ${status.getContainerId}. Total completed " +
+ s"$completedContainers.")
+ log.info(s"Diagnostics ${status.getDiagnostics}.")
+
+ messageListener foreach {
+ _ ! YarnMessage(s"Diagnostics for containerID=${status.getContainerId} in " +
+ s"state=${status.getState}.\n${status.getDiagnostics}")
+ }
+ }
+
+ if (allocatedContainers < numTaskManager) {
+ context.system.scheduler.scheduleOnce(ALLOCATION_DELAY, self, PollContainerCompletion)
+ } else if (completedContainers < numTaskManager) {
+ context.system.scheduler.scheduleOnce(COMPLETION_DELAY, self, PollContainerCompletion)
+ } else {
+ self ! StopYarnSession(FinalApplicationStatus.FAILED)
+ }
+ case None =>
+ log.error("The AMRMClient was not set.")
+ self ! StopYarnSession(FinalApplicationStatus.FAILED)
+ }
+ }
+
+ private def startYarnSession(conf: Configuration, actorSystemPort: Int, webServerPort: Int): Unit = {
+ Try {
log.info("Start yarn session.")
val memoryPerTaskManager = env.get(FlinkYarnClient.ENV_TM_MEMORY).toInt
val heapLimit = Utils.calculateHeapSize(memoryPerTaskManager)
@@ -134,7 +202,7 @@ trait YarnJobManager extends ActorLogMessages {
nmClientOption = Some(nm)
// Register with ResourceManager
- val url = s"http://$applicationMasterHost:$webServerport"
+ val url = s"http://$applicationMasterHost:$webServerPort"
log.info(s"Registering ApplicationMaster with tracking url $url.")
rm.registerApplicationMaster(applicationMasterHost, actorSystemPort, url)
@@ -146,7 +214,7 @@ trait YarnJobManager extends ActorLogMessages {
// Resource requirements for worker containers
val capability = Records.newRecord(classOf[Resource])
capability.setMemory(memoryPerTaskManager)
- capability.setVirtualCores(1) // hard-code that number (YARN is not accunting for CPUs)
+ capability.setVirtualCores(1) // hard-code that number (YARN is not accounting for CPUs)
// Make container requests to ResourceManager
for (i <- 0 until numTaskManager) {
@@ -193,60 +261,14 @@ trait YarnJobManager extends ActorLogMessages {
yarnClientUsername, conf, taskManagerLocalResources))
context.system.scheduler.scheduleOnce(ALLOCATION_DELAY, self, PollContainerCompletion)
-
- case PollContainerCompletion =>
- rmClientOption match {
- case Some(rmClient) =>
- val response = rmClient.allocate(completedContainers.toFloat / numTaskManager)
-
- for (container <- response.getAllocatedContainers.asScala) {
- log.info(s"Got new container for TM ${container.getId} on host ${
- container.getNodeId.getHost
- }")
-
- allocatedContainers += 1
-
- log.info(s"Launching container #$allocatedContainers.")
- nmClientOption match {
- case Some(nmClient) =>
- containerLaunchContext match {
- case Some(ctx) => nmClient.startContainer(container, ctx)
- case None =>
- log.error("The ContainerLaunchContext was not set.")
- self ! StopYarnSession(FinalApplicationStatus.FAILED)
- }
- case None =>
- log.error("The NMClient was not set.")
- self ! StopYarnSession(FinalApplicationStatus.FAILED)
- }
- }
-
- for (status <- response.getCompletedContainersStatuses.asScala) {
- completedContainers += 1
- log.info(s"Completed container ${status.getContainerId}. Total completed " +
- s"$completedContainers.")
- log.info(s"Diagnostics ${status.getDiagnostics}.")
-
- messageListener foreach {
- _ ! YarnMessage(s"Diagnostics for containerID=${status.getContainerId} in " +
- s"state=${status.getState}.\n${status.getDiagnostics}")
- }
- }
-
- if (allocatedContainers < numTaskManager) {
- context.system.scheduler.scheduleOnce(ALLOCATION_DELAY, self, PollContainerCompletion)
- } else if (completedContainers < numTaskManager) {
- context.system.scheduler.scheduleOnce(COMPLETION_DELAY, self, PollContainerCompletion)
- } else {
- self ! StopYarnSession(FinalApplicationStatus.FAILED)
- }
- case None =>
- log.error("The AMRMClient was not set.")
- self ! StopYarnSession(FinalApplicationStatus.FAILED)
- }
+ } recover {
+ case t: Throwable =>
+ log.error(t, "Could not start yarn session.")
+ self ! StopYarnSession(FinalApplicationStatus.FAILED)
+ }
}
- def createContainerLaunchContext(heapLimit: Int, hasLogback: Boolean, hasLog4j: Boolean,
+ private def createContainerLaunchContext(heapLimit: Int, hasLogback: Boolean, hasLog4j: Boolean,
yarnClientUsername: String, yarnConf: Configuration,
taskManagerLocalResources: Map[String, LocalResource]):
ContainerLaunchContext = {
@@ -294,12 +316,12 @@ trait YarnJobManager extends ActorLogMessages {
val securityTokens = ByteBuffer.wrap(dob.getData, 0, dob.getLength)
ctx.setTokens(securityTokens)
} catch {
- case e: IOException =>
- log.warning("Getting current user info failed when trying to launch the container", e)
+ case t: Throwable =>
+ log.error(t, "Getting current user info failed when trying to launch the container")
}
ctx
}
- def env = System.getenv()
+ private def env = System.getenv()
}
[2/3] flink git commit: [FLINK-1554] [runtime] Allows the
LocalFlinkMiniCluster to start multiple TaskManager in the same ActorSystem.
Posted by tr...@apache.org.
[FLINK-1554] [runtime] Allows the LocalFlinkMiniCluster to start multiple TaskManager in the same ActorSystem.
This closes #403.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/649f1583
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/649f1583
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/649f1583
Branch: refs/heads/master
Commit: 649f158391aebcefcaabcdaa02d5c8b95be45777
Parents: 447ce0a
Author: Till Rohrmann <tr...@apache.org>
Authored: Mon Feb 16 15:04:42 2015 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Feb 17 11:53:36 2015 +0100
----------------------------------------------------------------------
.../flink/runtime/jobmanager/JobManager.scala | 2 +-
.../runtime/minicluster/FlinkMiniCluster.scala | 11 +++
.../minicluster/LocalFlinkMiniCluster.scala | 7 ++
.../flink/runtime/taskmanager/TaskManager.scala | 31 ++++---
.../runtime/testingUtils/TestingCluster.scala | 3 +-
flink-tests/pom.xml | 5 ++
.../LocalFlinkMiniClusterITCase.java | 88 ++++++++++++++++++++
.../scala/org/apache/flink/yarn/YarnUtils.scala | 5 +-
8 files changed, 136 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/649f1583/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index b490b32..bfce7a2 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -664,7 +664,7 @@ object JobManager {
if(executionMode.equals(LOCAL)){
LOG.info("Starting embedded TaskManager for JobManager's LOCAL mode execution")
- TaskManager.startActorWithConfiguration("", configuration,
+ TaskManager.startActorWithConfiguration("", TaskManager.TASK_MANAGER_NAME, configuration,
localAkkaCommunication = false, localTaskManagerCommunication = true)(jobManagerSystem)
}
http://git-wip-us.apache.org/repos/asf/flink/blob/649f1583/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
index 6eea21c..3eb1d1e 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/FlinkMiniCluster.scala
@@ -19,6 +19,7 @@
package org.apache.flink.runtime.minicluster
import java.net.InetAddress
+import akka.pattern.Patterns.gracefulStop
import akka.pattern.ask
import akka.actor.{ActorRef, ActorSystem}
@@ -131,6 +132,16 @@ abstract class FlinkMiniCluster(userConfiguration: Configuration,
}
def shutdown(): Unit = {
+ val futures = taskManagerActors map {
+ gracefulStop(_, timeout)
+ }
+
+ val future = gracefulStop(jobManagerActor, timeout)
+
+ implicit val executionContext = AkkaUtils.globalExecutionContext
+
+ Await.ready(Future.sequence(future +: futures), timeout)
+
if(!singleActorSystem){
taskManagerActorSystems foreach {
_.shutdown()
http://git-wip-us.apache.org/repos/asf/flink/blob/649f1583/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
index 8b16969..88006ac 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/minicluster/LocalFlinkMiniCluster.scala
@@ -90,7 +90,14 @@ class LocalFlinkMiniCluster(userConfiguration: Configuration, singleActorSystem:
val localExecution = numTaskManagers == 1
+ val taskManagerName = if(singleActorSystem) {
+ TaskManager.TASK_MANAGER_NAME + "_" + (index + 1)
+ } else {
+ TaskManager.TASK_MANAGER_NAME
+ }
+
TaskManager.startActorWithConfiguration(HOSTNAME,
+ taskManagerName,
config,
singleActorSystem,
localExecution)(system)
http://git-wip-us.apache.org/repos/asf/flink/blob/649f1583/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 5a647d1..6d610a4 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -711,18 +711,19 @@ object TaskManager {
LOG.info("Security is enabled. Starting secure TaskManager.")
SecurityUtils.runSecured(new FlinkSecuredRunner[Unit] {
override def run(): Unit = {
- startActor(hostname, port, configuration)
+ startActor(hostname, port, configuration, TaskManager.TASK_MANAGER_NAME)
}
})
} else {
- startActor(hostname, port, configuration)
+ startActor(hostname, port, configuration, TaskManager.TASK_MANAGER_NAME)
}
}
- def startActor(hostname: String, port: Int, configuration: Configuration) : Unit = {
+ def startActor(hostname: String, port: Int, configuration: Configuration,
+ taskManagerName: String) : Unit = {
val (taskManagerSystem, _) = startActorSystemAndActor(hostname, port, configuration,
- localAkkaCommunication = false, localTaskManagerCommunication = false)
+ taskManagerName, localAkkaCommunication = false, localTaskManagerCommunication = false)
taskManagerSystem.awaitTermination()
}
@@ -780,6 +781,7 @@ object TaskManager {
}
def startActorSystemAndActor(hostname: String, port: Int, configuration: Configuration,
+ taskManagerName: String,
localAkkaCommunication: Boolean,
localTaskManagerCommunication: Boolean): (ActorSystem, ActorRef) = {
implicit val actorSystem = AkkaUtils.createActorSystem(configuration, Some((hostname, port)))
@@ -788,7 +790,7 @@ object TaskManager {
parseConfiguration(hostname, configuration, localAkkaCommunication,
localTaskManagerCommunication)
- (actorSystem, startActor(connectionInfo, jobManagerURL, taskManagerConfig,
+ (actorSystem, startActor(taskManagerName, connectionInfo, jobManagerURL, taskManagerConfig,
networkConfig))
}
@@ -916,19 +918,23 @@ object TaskManager {
(connectionInfo, jobManagerURL, taskManagerConfig, networkConfig)
}
- def startActor(connectionInfo: InstanceConnectionInfo, jobManagerURL: String,
+ def startActor(taskManagerName: String,
+ connectionInfo: InstanceConnectionInfo,
+ jobManagerURL: String,
taskManagerConfig: TaskManagerConfiguration,
networkConfig: NetworkEnvironmentConfiguration)
(implicit actorSystem: ActorSystem): ActorRef = {
- startActor(Props(new TaskManager(connectionInfo, jobManagerURL, taskManagerConfig,
- networkConfig)))
+ startActor(taskManagerName,
+ Props(new TaskManager(connectionInfo, jobManagerURL, taskManagerConfig, networkConfig)))
}
- def startActor(props: Props)(implicit actorSystem: ActorSystem): ActorRef = {
- actorSystem.actorOf(props, TASK_MANAGER_NAME)
+ def startActor(taskManagerName: String, props: Props)
+ (implicit actorSystem: ActorSystem): ActorRef = {
+ actorSystem.actorOf(props, taskManagerName)
}
- def startActorWithConfiguration(hostname: String, configuration: Configuration,
+ def startActorWithConfiguration(hostname: String, taskManagerName: String,
+ configuration: Configuration,
localAkkaCommunication: Boolean,
localTaskManagerCommunication: Boolean)
(implicit system: ActorSystem) = {
@@ -936,7 +942,8 @@ object TaskManager {
parseConfiguration(hostname, configuration, localAkkaCommunication,
localTaskManagerCommunication)
- startActor(connectionInfo, jobManagerURL, taskManagerConfig, networkConnectionConfiguration)
+ startActor(taskManagerName, connectionInfo, jobManagerURL, taskManagerConfig,
+ networkConnectionConfiguration)
}
def startProfiler(instancePath: String, reportInterval: Long)(implicit system: ActorSystem):
http://git-wip-us.apache.org/repos/asf/flink/blob/649f1583/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
index e2660d5..9d132b6 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
@@ -68,7 +68,8 @@ FlinkMiniCluster(userConfiguration, singleActorSystem) {
localAkkaCommunication = singleActorSystem, localTaskManagerCommunication = true)
system.actorOf(Props(new TaskManager(connectionInfo, jobManagerURL, taskManagerConfig,
- networkConnectionConfig) with TestingTaskManager), TaskManager.TASK_MANAGER_NAME + index)
+ networkConnectionConfig) with TestingTaskManager), TaskManager.TASK_MANAGER_NAME + "_" +
+ (index + 1))
}
def restartJobManager(): Unit = {
http://git-wip-us.apache.org/repos/asf/flink/blob/649f1583/flink-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index 985c675..33ca642 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -135,6 +135,11 @@ under the License.
<artifactId>scalatest_2.10</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>com.typesafe.akka</groupId>
+ <artifactId>akka-testkit_2.10</artifactId>
+ </dependency>
<dependency>
<groupId>joda-time</groupId>
http://git-wip-us.apache.org/repos/asf/flink/blob/649f1583/flink-tests/src/test/java/org/apache/flink/test/runtime/minicluster/LocalFlinkMiniClusterITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/minicluster/LocalFlinkMiniClusterITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/minicluster/LocalFlinkMiniClusterITCase.java
new file mode 100644
index 0000000..7933932
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/minicluster/LocalFlinkMiniClusterITCase.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.runtime.minicluster;
+
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.testkit.JavaTestKit;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.messages.JobManagerMessages;
+import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class LocalFlinkMiniClusterITCase {
+
+ static ActorSystem system;
+
+ @BeforeClass
+ public static void setup() {
+ system = ActorSystem.create("Testkit", AkkaUtils.getDefaultAkkaConfig());
+ }
+
+ @AfterClass
+ public static void teardown() {
+ JavaTestKit.shutdownActorSystem(system);
+ system = null;
+ }
+
+ @Test
+ public void testLocalFlinkMiniClusterWithMultipleTaskManagers() {
+ LocalFlinkMiniCluster miniCluster = null;
+
+ final int numTMs = 3;
+ final int numSlots = 14;
+
+ try{
+ Configuration config = new Configuration();
+ config.setInteger(ConfigConstants.LOCAL_INSTANCE_MANAGER_NUMBER_TASK_MANAGER, numTMs);
+ config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, numSlots);
+ miniCluster = new LocalFlinkMiniCluster(config, true);
+
+ final ActorRef jm = miniCluster.getJobManager();
+
+ new JavaTestKit(system) {{
+ new Within(TestingUtils.TESTING_DURATION()) {
+
+ @Override
+ protected void run() {
+ jm.tell(JobManagerMessages.getRequestNumberRegisteredTaskManager(),
+ getRef());
+
+ expectMsgEquals(TestingUtils.TESTING_DURATION(), numTMs);
+
+ jm.tell(JobManagerMessages.getRequestTotalNumberOfSlots(), getRef());
+
+ expectMsgEquals(TestingUtils.TESTING_DURATION(), numTMs*numSlots);
+ }
+ };
+ }};
+
+
+ } finally {
+ if (miniCluster != null) {
+ miniCluster.stop();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/649f1583/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnUtils.scala
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnUtils.scala b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnUtils.scala
index 775fcd0..66da8ec 100644
--- a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnUtils.scala
+++ b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnUtils.scala
@@ -38,7 +38,8 @@ object YarnUtils {
TaskManager.parseConfiguration(hostname, config, localAkkaCommunication = false,
localTaskManagerCommunication = false)
- (actorSystem, TaskManager.startActor(Props(new TaskManager(connectionInfo, jobManagerURL,
- taskManagerConfig, networkConnectionConfiguration) with YarnTaskManager))(actorSystem))
+ (actorSystem, TaskManager.startActor(TaskManager.TASK_MANAGER_NAME,
+ Props(new TaskManager(connectionInfo, jobManagerURL, taskManagerConfig,
+ networkConnectionConfiguration) with YarnTaskManager))(actorSystem))
}
}
[3/3] flink git commit: [FLINK-1556] [runtime] Corrects faulty
JobClient behaviour in case of a submission failure
Posted by tr...@apache.org.
[FLINK-1556] [runtime] Corrects faulty JobClient behaviour in case of a submission failure
Fixes unhandled FailsIntermediateResultPartitions message. Changes JobResultFailed and JobResultCanceled to send the cause as a throwable instead of a string.
Fixes NullPointerException if an execution has already been prepared for archiving and sendPartitionInfos is called asynchronously
This closes #406.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ac7bf427
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ac7bf427
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ac7bf427
Branch: refs/heads/master
Commit: ac7bf427900e60abef3ff4344a5873349e6d108e
Parents: 649f158
Author: Till Rohrmann <tr...@apache.org>
Authored: Mon Feb 16 10:17:21 2015 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Tue Feb 17 12:19:05 2015 +0100
----------------------------------------------------------------------
.../org/apache/flink/client/program/Client.java | 40 +++++-----
.../flink/client/CliFrontendListCancelTest.java | 11 +--
.../src/test/resources/log4j-test.properties | 10 ++-
.../client/JobCancellationException.java | 30 ++++++++
.../runtime/client/JobExecutionException.java | 35 ++-------
.../runtime/client/JobTimeoutException.java | 29 +++++++
.../runtime/execution/CancelTaskException.java | 8 ++
.../runtime/execution/RuntimeEnvironment.java | 2 +-
.../flink/runtime/executiongraph/Execution.java | 80 ++++++++++----------
.../runtime/executiongraph/ExecutionGraph.java | 20 +++--
.../apache/flink/runtime/client/JobClient.scala | 52 +++++++++----
.../flink/runtime/jobmanager/JobManager.scala | 48 ++----------
.../runtime/jobmanager/JobManagerProfiler.scala | 8 ++
.../runtime/jobmanager/MemoryArchivist.scala | 8 ++
.../messages/ExecutionGraphMessages.scala | 4 +-
.../runtime/messages/JobmanagerMessages.scala | 8 +-
.../flink/runtime/taskmanager/TaskManager.scala | 14 ++++
.../taskmanager/TaskManagerProfiler.scala | 8 ++
.../src/test/resources/log4j-test.properties | 2 +-
.../runtime/jobmanager/JobManagerITCase.scala | 2 +-
.../test/cancelling/CancellingTestBase.java | 10 +--
.../apache/flink/yarn/ApplicationClient.scala | 7 ++
.../org/apache/flink/yarn/YarnJobManager.scala | 9 ++-
23 files changed, 265 insertions(+), 180 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/Client.java b/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
index 04e6c4d..1fc7696 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
@@ -45,8 +45,10 @@ import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.GlobalConfiguration;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.client.JobCancellationException;
import org.apache.flink.runtime.client.JobClient;
import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.client.JobTimeoutException;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.messages.JobManagerMessages.SubmissionFailure;
import org.apache.flink.runtime.messages.JobManagerMessages.SubmissionResponse;
@@ -331,7 +333,11 @@ public class Client {
try {
JobClient.uploadJarFiles(jobGraph, hostname, client, timeout);
+ } catch (IOException e) {
+ throw new ProgramInvocationException("Could not upload the programs JAR files to the JobManager.", e);
+ }
+ try{
if (wait) {
return JobClient.submitJobAndWait(jobGraph, printStatusDuringExecution, client, timeout);
}
@@ -340,30 +346,20 @@ public class Client {
if (response instanceof SubmissionFailure) {
SubmissionFailure failure = (SubmissionFailure) response;
throw new ProgramInvocationException(
- "Failed to submit the job to the flink JobManager", failure.cause());
+ "Failed to submit the job to the JobManager.", failure.cause());
}
}
- }
- catch (IOException e) {
- throw new ProgramInvocationException("Could not upload the programs JAR files to the JobManager.", e);
- }
- catch (JobExecutionException e) {
- if (e.isJobCanceledByUser()) {
- throw new ProgramInvocationException("The program has been canceled.");
- }
- else if (e.isConnectionTimedOut()) {
- Throwable ae = null; //getAssociationError(monitoredErrors);
- String message = ae == null ? "." : ": " + ae.getMessage();
- throw new ProgramInvocationException("Lost connection to the JobManager" + message);
- }
- else {
- throw new ProgramInvocationException("The program execution failed: " + e.getMessage());
- }
- }
- catch (Exception e) {
- Throwable ae = null; //getAssociationError(monitoredErrors);
- String message = ae == null ? "." : ": " + ae.getMessage();
- throw new ProgramInvocationException("Connection to JobManager failed" + message);
+ } catch (JobExecutionException e) {
+ throw new ProgramInvocationException("The program execution failed.", e);
+ } catch (JobTimeoutException e) {
+ throw new ProgramInvocationException("Lost connection to the JobManager.", e);
+ } catch (JobCancellationException e) {
+ throw new ProgramInvocationException("The program has been canceled.", e);
+ } catch (ProgramInvocationException e) {
+ // forward exception resulting from submission failure
+ throw e;
+ } catch (Exception e) {
+ throw new ProgramInvocationException("Exception occurred during job execution.", e);
}
finally {
actorSystem.shutdown();
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-clients/src/test/java/org/apache/flink/client/CliFrontendListCancelTest.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendListCancelTest.java b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendListCancelTest.java
index 9dc13ba..fafe929 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/CliFrontendListCancelTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/CliFrontendListCancelTest.java
@@ -94,8 +94,6 @@ public class CliFrontendListCancelTest {
@Test
public void testList() {
try {
- final ActorRef jm = actorSystem.actorOf(Props.create(CliJobManager.class, (Object)null));
-
// test unrecognized option
{
String[] parameters = {"-v", "-k"};
@@ -104,16 +102,9 @@ public class CliFrontendListCancelTest {
assertTrue(retCode == 1);
}
- // test missing flags
- {
- String[] parameters = {};
- CliFrontend testFrontend = new CliFrontendTestUtils.TestingCliFrontend();
- int retCode = testFrontend.list(parameters);
- assertTrue(retCode != 0);
- }
-
// test list properly
{
+ final ActorRef jm = actorSystem.actorOf(Props.create(CliJobManager.class, (Object)null));
String[] parameters = {"-r", "-s"};
InfoListTestCliFrontend testFrontend = new InfoListTestCliFrontend(jm);
int retCode = testFrontend.list(parameters);
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-clients/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-clients/src/test/resources/log4j-test.properties b/flink-clients/src/test/resources/log4j-test.properties
index 2fb9345..04ec355 100644
--- a/flink-clients/src/test/resources/log4j-test.properties
+++ b/flink-clients/src/test/resources/log4j-test.properties
@@ -16,4 +16,12 @@
# limitations under the License.
################################################################################
-log4j.rootLogger=OFF
\ No newline at end of file
+# Set root logger level to OFF and its only appender to A1.
+log4j.rootLogger=OFF, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobCancellationException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobCancellationException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobCancellationException.java
new file mode 100644
index 0000000..1a9a1d0
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobCancellationException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.client;
+
+/**
+ * An exception which is thrown by the JobClient if a job is aborted as a result of a user
+ * cancellation.
+ */
+public class JobCancellationException extends Exception {
+
+ public JobCancellationException(final String msg, final Throwable cause){
+ super(msg, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobExecutionException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobExecutionException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobExecutionException.java
index f35f412..d5eb492 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobExecutionException.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobExecutionException.java
@@ -19,43 +19,20 @@
package org.apache.flink.runtime.client;
/**
- * This exception is thrown by the {@link JobClient} if a job has been aborted either as a result of a user
- * request or an error which occurred during the execution.
+ * This exception is thrown by the {@link JobClient} if a job has been aborted as a result of an
+ * error which occurred during the execution.
*/
public class JobExecutionException extends Exception {
- public static enum ExecutionErrorCause {
- CANCELED,
- TIMEOUT_TO_JOB_MANAGER,
- ERROR
- }
-
- // ------------------------------------------------------------------------
-
private static final long serialVersionUID = 2818087325120827525L;
- private final ExecutionErrorCause cause;
-
/**
* Constructs a new job execution exception.
*
- * @param msg The message that shall be encapsulated by this exception.
- * @param cause The cause for the execution exception.
+ * @param msg The cause for the execution exception.
+ * @param cause The cause of the exception
*/
- public JobExecutionException(String msg, ExecutionErrorCause cause) {
- super(msg);
- this.cause = cause;
- }
-
- public boolean isJobCanceledByUser() {
- return cause == ExecutionErrorCause.CANCELED;
- }
-
- public boolean isConnectionTimedOut() {
- return cause == ExecutionErrorCause.TIMEOUT_TO_JOB_MANAGER;
- }
-
- public boolean isError() {
- return cause == ExecutionErrorCause.ERROR;
+ public JobExecutionException(String msg, Throwable cause) {
+ super(msg, cause);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobTimeoutException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobTimeoutException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobTimeoutException.java
new file mode 100644
index 0000000..2bd6ec5
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/client/JobTimeoutException.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.client;
+
+/**
+ * An exception which is thrown by the JobClient if the job manager is no longer reachable.
+ */
+public class JobTimeoutException extends Exception {
+
+ public JobTimeoutException(final String msg, final Throwable cause) {
+ super(msg, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/java/org/apache/flink/runtime/execution/CancelTaskException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/CancelTaskException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/CancelTaskException.java
index 45dc411..959ab07 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/CancelTaskException.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/CancelTaskException.java
@@ -23,4 +23,12 @@ package org.apache.flink.runtime.execution;
*/
public class CancelTaskException extends RuntimeException {
private static final long serialVersionUID = 1L;
+
+ public CancelTaskException(final String msg) {
+ super(msg);
+ }
+
+ public CancelTaskException() {
+ super("");
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/java/org/apache/flink/runtime/execution/RuntimeEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/RuntimeEnvironment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/RuntimeEnvironment.java
index 6be0397..f78ea92 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/RuntimeEnvironment.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/RuntimeEnvironment.java
@@ -205,7 +205,7 @@ public class RuntimeEnvironment implements Environment, Runnable {
// Make sure, we enter the catch block when the task has been canceled
if (owner.isCanceledOrFailed()) {
- throw new CancelTaskException();
+ throw new CancelTaskException("Task has been canceled or failed");
}
// Finish the produced partitions
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index 83a25a7..56f9416 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -601,7 +601,7 @@ public class Execution implements Serializable {
// check if the ExecutionVertex has already been archived and thus cleared the
// partial partition infos queue
- if(partialPartitionInfos != null) {
+ if(partialPartitionInfos != null && !partialPartitionInfos.isEmpty()) {
PartialPartitionInfo partialPartitionInfo;
@@ -730,64 +730,64 @@ public class Execution implements Serializable {
private void sendCancelRpcCall() {
final SimpleSlot slot = this.assignedResource;
- if (slot == null) {
- return;
- }
- Future<Object> cancelResult = AkkaUtils.retry(slot.getInstance().getTaskManager(), new
- TaskManagerMessages.CancelTask(attemptId), NUM_CANCEL_CALL_TRIES,
- AkkaUtils.globalExecutionContext(), timeout);
+ if (slot != null) {
- cancelResult.onComplete(new OnComplete<Object>(){
+ Future<Object> cancelResult = AkkaUtils.retry(slot.getInstance().getTaskManager(), new
+ TaskManagerMessages.CancelTask(attemptId), NUM_CANCEL_CALL_TRIES,
+ AkkaUtils.globalExecutionContext(), timeout);
- @Override
- public void onComplete(Throwable failure, Object success) throws Throwable {
- if (failure != null) {
- fail(new Exception("Task could not be canceled.", failure));
- } else {
- TaskOperationResult result = (TaskOperationResult)success;
- if(!result.success()){
- LOG.debug("Cancel task call did not find task. Probably akka message call" +
- " race.");
+ cancelResult.onComplete(new OnComplete<Object>() {
+
+ @Override
+ public void onComplete(Throwable failure, Object success) throws Throwable {
+ if (failure != null) {
+ fail(new Exception("Task could not be canceled.", failure));
+ } else {
+ TaskOperationResult result = (TaskOperationResult) success;
+ if (!result.success()) {
+ LOG.debug("Cancel task call did not find task. Probably akka message call" +
+ " race.");
+ }
}
}
- }
- }, AkkaUtils.globalExecutionContext());
+ }, AkkaUtils.globalExecutionContext());
+ }
}
private void sendFailIntermediateResultPartitionsRPCCall() {
final SimpleSlot slot = this.assignedResource;
- if (slot == null) {
- return;
- }
- final Instance instance = slot.getInstance();
+ if (slot != null) {
+ final Instance instance = slot.getInstance();
- if (instance.isAlive()) {
- try {
- // TODO For some tests this could be a problem when querying too early if all resources were released
- instance.getTaskManager().tell(new TaskManagerMessages.FailIntermediateResultPartitions(attemptId), ActorRef.noSender());
- }
- catch (Throwable t) {
- fail(new Exception("Intermediate result partition could not be failed.", t));
+ if (instance.isAlive()) {
+ try {
+ // TODO For some tests this could be a problem when querying too early if all resources were released
+ instance.getTaskManager().tell(new TaskManagerMessages.FailIntermediateResultPartitions(attemptId), ActorRef.noSender());
+ } catch (Throwable t) {
+ fail(new Exception("Intermediate result partition could not be failed.", t));
+ }
}
}
}
private void sendUpdateTaskRpcCall(final SimpleSlot consumerSlot,
final TaskManagerMessages.UpdateTask updateTaskMsg) {
- final Instance instance = consumerSlot.getInstance();
+ if (consumerSlot != null) {
+ final Instance instance = consumerSlot.getInstance();
- Future<Object> futureUpdate = Patterns.ask(instance.getTaskManager(), updateTaskMsg,
- new Timeout(timeout));
+ Future<Object> futureUpdate = Patterns.ask(instance.getTaskManager(), updateTaskMsg,
+ new Timeout(timeout));
- futureUpdate.onFailure(new OnFailure() {
- @Override
- public void onFailure(Throwable failure) throws Throwable {
- fail(new IllegalStateException("Update task on instance " + instance +
- " failed due to:", failure));
- }
- }, AkkaUtils.globalExecutionContext());
+ futureUpdate.onFailure(new OnFailure() {
+ @Override
+ public void onFailure(Throwable failure) throws Throwable {
+ fail(new IllegalStateException("Update task on instance " + instance +
+ " failed due to:", failure));
+ }
+ }, AkkaUtils.globalExecutionContext());
+ }
}
// --------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
index c1f45e1..7198076 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
@@ -398,10 +398,15 @@ public class ExecutionGraph implements Serializable {
}
else if (transitionState(current, JobStatus.FAILING, t)) {
this.failureCause = t;
-
- // cancel all. what is failed will not cancel but stay failed
- for (ExecutionJobVertex ejv : verticesInCreationOrder) {
- ejv.cancel();
+
+ if (!verticesInCreationOrder.isEmpty()) {
+ // cancel all. what is failed will not cancel but stay failed
+ for (ExecutionJobVertex ejv : verticesInCreationOrder) {
+ ejv.cancel();
+ }
+ } else {
+ // set the state of the job to failed
+ transitionState(current, JobStatus.FAILED, t);
}
return;
@@ -568,15 +573,18 @@ public class ExecutionGraph implements Serializable {
this.executionListenerActors.add(listener);
}
+ public boolean containsJobStatusListener(ActorRef listener) {
+ return this.jobStatusListenerActors.contains(listener);
+ }
+
/**
* NOTE: This method never throws an error, only logs errors caused by the notified listeners.
*/
private void notifyJobStatusChange(JobStatus newState, Throwable error) {
if(jobStatusListenerActors.size() > 0){
- String message = error == null ? null : ExceptionUtils.stringifyException(error);
for(ActorRef listener: jobStatusListenerActors){
listener.tell(new ExecutionGraphMessages.JobStatusChanged(jobID, newState, System.currentTimeMillis(),
- message), ActorRef.noSender());
+ error), ActorRef.noSender());
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/scala/org/apache/flink/runtime/client/JobClient.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/client/JobClient.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/client/JobClient.scala
index 54291a8..4a64f87 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/client/JobClient.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/client/JobClient.scala
@@ -48,16 +48,28 @@ Actor with ActorLogMessages with ActorLogging {
override def receiveWithLogMessages: Receive = {
case SubmitJobDetached(jobGraph) =>
jobManager forward SubmitJob(jobGraph, registerForEvents = false, detached = true)
+
case cancelJob: CancelJob =>
jobManager forward cancelJob
+
case SubmitJobAndWait(jobGraph, listen) =>
val listener = context.actorOf(Props(classOf[JobClientListener], sender))
jobManager.tell(SubmitJob(jobGraph, registerForEvents = listen, detached = false), listener)
+
case RequestBlobManagerPort =>
jobManager forward RequestBlobManagerPort
+
case RequestJobManagerStatus =>
jobManager forward RequestJobManagerStatus
}
+
+ /**
+ * Handle unmatched messages with an exception.
+ */
+ override def unhandled(message: Any): Unit = {
+ // let the actor crash
+ throw new RuntimeException("Received unknown message " + message)
+ }
}
/**
@@ -70,9 +82,9 @@ Actor with ActorLogMessages with ActorLogging {
class JobClientListener(jobSubmitter: ActorRef) extends Actor with ActorLogMessages with
ActorLogging {
override def receiveWithLogMessages: Receive = {
- case SubmissionFailure(_, t) =>
- jobSubmitter ! Failure(t)
- self ! PoisonPill
+ case SubmissionFailure(jobID, t) =>
+ System.out.println(s"Submission of job with ID $jobID was unsuccessful, " +
+ s"because ${t.getMessage}.")
case SubmissionSuccess(_) =>
@@ -80,20 +92,26 @@ ActorLogging {
jobSubmitter ! new JobExecutionResult(duration, accumulatorResults)
self ! PoisonPill
- case JobResultCanceled(_, msg) =>
- jobSubmitter ! Failure(
- new JobExecutionException(msg, JobExecutionException.ExecutionErrorCause.CANCELED))
+ case JobResultCanceled(_, t) =>
+ jobSubmitter ! Failure(new JobCancellationException("The job has been cancelled.", t))
self ! PoisonPill
- case JobResultFailed(_, msg) =>
- jobSubmitter ! Failure(new JobExecutionException(msg,
- JobExecutionException.ExecutionErrorCause.ERROR))
+ case JobResultFailed(_, t) =>
+ jobSubmitter ! Failure(new JobExecutionException("The job execution failed.", t))
self ! PoisonPill
case msg =>
// we have to use System.out.println here to avoid erroneous behavior for output redirection
System.out.println(msg.toString)
}
+
+ /**
+ * Handle unmatched messages with an exception.
+ */
+ override def unhandled(message: Any): Unit = {
+ // let the actor crash
+ throw new RuntimeException("Received unknown message " + message)
+ }
}
/**
@@ -193,7 +211,7 @@ object JobClient {
* execution fails.
* @return The job execution result
*/
- @throws(classOf[JobExecutionException])
+ @throws(classOf[Exception])
def submitJobAndWait(jobGraph: JobGraph, listenToStatusEvents: Boolean, jobClient: ActorRef)
(implicit timeout: FiniteDuration): JobExecutionResult = {
@@ -215,9 +233,7 @@ object JobClient {
Await.result(jmStatus, timeout)
} catch {
case t: Throwable =>
- throw new JobExecutionException(
- "JobManager not reachable anymore. Terminate waiting for job answer.",
- JobExecutionException.ExecutionErrorCause.TIMEOUT_TO_JOB_MANAGER)
+ throw new JobTimeoutException("Lost connection to job manager.", t)
}
}
}
@@ -236,11 +252,19 @@ object JobClient {
* @param timeout Tiemout for futures
* @return The submission response
*/
+ @throws(classOf[Exception])
def submitJobDetached(jobGraph: JobGraph, jobClient: ActorRef)(implicit timeout: FiniteDuration):
SubmissionResponse = {
val response = (jobClient ? SubmitJobDetached(jobGraph))(timeout)
- Await.result(response.mapTo[SubmissionResponse], timeout)
+ try {
+ Await.result(response.mapTo[SubmissionResponse], timeout)
+ } catch {
+ case timeout: TimeoutException =>
+ throw new JobTimeoutException("Timeout while waiting for the submission result.", timeout);
+ case t: Throwable =>
+ throw new JobExecutionException("Exception while waiting for the submission result.", t)
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index bfce7a2..1741cdb 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -50,7 +50,6 @@ import org.apache.flink.util.InstantiationUtil
import org.slf4j.LoggerFactory
import akka.actor._
-import akka.pattern.ask
import scala.concurrent._
import scala.concurrent.duration._
@@ -103,9 +102,6 @@ class JobManager(val configuration: Configuration,
// List of current jobs running
val currentJobs = scala.collection.mutable.HashMap[JobID, (ExecutionGraph, JobInfo)]()
- // Map of actors which want to be notified once a specific job terminates
- val finalJobStatusListener = scala.collection.mutable.HashMap[JobID, Set[ActorRef]]()
-
override def preStart(): Unit = {
LOG.info(s"Starting JobManager at ${self.path}.")
@@ -261,9 +257,9 @@ class JobManager(val configuration: Configuration,
case JobStatusChanged(jobID, newJobStatus, timeStamp, optionalMessage) =>
currentJobs.get(jobID) match {
case Some((executionGraph, jobInfo)) => executionGraph.getJobName
- log.info("Status of job {} ({}) changed to {}{}.",
+ log.info("Status of job {} ({}) changed to {} {}.",
jobID, executionGraph.getJobName, newJobStatus,
- if(optionalMessage == null) "" else optionalMessage)
+ if(optionalMessage == null) "" else optionalMessage.getMessage)
if(newJobStatus.isTerminalState) {
jobInfo.end = timeStamp
@@ -279,14 +275,9 @@ class JobManager(val configuration: Configuration,
case JobStatus.FAILED =>
jobInfo.client ! JobResultFailed(jobID, optionalMessage)
case x =>
- jobInfo.client ! JobResultFailed(jobID, s"$x is not a terminal state.")
- throw new IllegalStateException(s"$x is not a terminal state.")
- }
- }
-
- finalJobStatusListener.get(jobID) foreach {
- _ foreach {
- _ ! CurrentJobStatus(jobID, newJobStatus)
+ val exception = new IllegalStateException(s"$x is not a terminal state.")
+ jobInfo.client ! JobResultFailed(jobID, exception)
+ throw exception
}
}
@@ -296,16 +287,6 @@ class JobManager(val configuration: Configuration,
removeJob(jobID)
}
- case RequestFinalJobStatus(jobID) =>
- currentJobs.get(jobID) match {
- case Some(_) =>
- val listeners = finalJobStatusListener.getOrElse(jobID, Set())
- finalJobStatusListener += jobID -> (listeners + sender)
- case None =>
- // There is no job running with this job ID. Check the archive.
- archive forward RequestJobStatus(jobID)
- }
-
case ScheduleOrUpdateConsumers(jobId, executionId, partitionIndex) =>
currentJobs.get(jobId) match {
case Some((executionGraph, _)) =>
@@ -497,25 +478,12 @@ class JobManager(val configuration: Configuration,
* before. That way the proper cleanup of the job is triggered in the JobStatusChanged
* handler.
*/
- val status = (self ? RequestFinalJobStatus(jobGraph.getJobID))(10 second)
-
- /*
- * if we cannot register as final job status listener, then send manually a
- * JobStatusChanged message with JobStatus.FAILED.
- */
- val selfActorRef = self
- status.onFailure{
- case _: Throwable => selfActorRef ! JobStatusChanged(executionGraph.getJobID,
- JobStatus.FAILED, System.currentTimeMillis(), s"Cleanup job ${jobGraph.getJobID}.")
+ if (!executionGraph.containsJobStatusListener(self)) {
+ executionGraph.registerJobStatusListener(self)
}
- /*
- * Don't send the client the final job status because we will send him a
- * SubmissionFailure.
- */
- jobInfo.detached = true
-
executionGraph.fail(t)
+
case None =>
libraryCacheManager.unregisterJob(jobGraph.getJobID)
currentJobs.remove(jobGraph.getJobID)
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManagerProfiler.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManagerProfiler.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManagerProfiler.scala
index e44f7e9..eb8f913 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManagerProfiler.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManagerProfiler.scala
@@ -41,4 +41,12 @@ class JobManagerProfiler extends Actor with ActorLogMessages with ActorLogging w
log.error(s"Received unknown profiling data: ${x.getClass.getName}" )
}
}
+
+ /**
+ * Handle unmatched messages with an exception.
+ */
+ override def unhandled(message: Any): Unit = {
+ // let the actor crash
+ throw new RuntimeException("Received unknown message " + message)
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/MemoryArchivist.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/MemoryArchivist.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/MemoryArchivist.scala
index 2d055ed..0c3384c 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/MemoryArchivist.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/MemoryArchivist.scala
@@ -82,6 +82,14 @@ ActorLogging {
}
/**
+ * Handle unmatched messages with an exception.
+ */
+ override def unhandled(message: Any): Unit = {
+ // let the actor crash
+ throw new RuntimeException("Received unknown message " + message)
+ }
+
+ /**
* Gets all graphs that have not been garbage collected.
* @return An iterable with all valid ExecutionGraphs
*/
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/ExecutionGraphMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/ExecutionGraphMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/ExecutionGraphMessages.scala
index ef5b99c..d413d13 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/ExecutionGraphMessages.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/ExecutionGraphMessages.scala
@@ -66,10 +66,10 @@ object ExecutionGraphMessages {
* @param jobID identifying the correspong job
* @param newJobStatus
* @param timestamp
- * @param optionalMessage
+ * @param error
*/
case class JobStatusChanged(jobID: JobID, newJobStatus: JobStatus, timestamp: Long,
- optionalMessage: String){
+ error: Throwable){
override def toString: String = {
s"${timestampToString(timestamp)}\tJob execution switched to status $newJobStatus."
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/JobmanagerMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/JobmanagerMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/JobmanagerMessages.scala
index c270d33..f25083f 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/JobmanagerMessages.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/JobmanagerMessages.scala
@@ -196,16 +196,16 @@ object JobManagerMessages {
/**
* Denotes a cancellation of the job.
* @param jobID
- * @param msg
+ * @param t
*/
- case class JobResultCanceled(jobID: JobID, msg: String) extends JobResult
+ case class JobResultCanceled(jobID: JobID, t: Throwable) extends JobResult
/**
* Denotes a failed job execution.
* @param jobID
- * @param msg
+ * @param t
*/
- case class JobResultFailed(jobID: JobID, msg:String) extends JobResult
+ case class JobResultFailed(jobID: JobID, t: Throwable) extends JobResult
sealed trait SubmissionResponse{
def jobID: JobID
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 6d610a4..99d824b 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -354,6 +354,20 @@ import scala.collection.JavaConverters._
cleanupTaskManager()
tryJobManagerRegistration()
+
+ case FailIntermediateResultPartitions(executionID) =>
+ log.info("Fail intermediate result partitions associated with execution {}.", executionID)
+ networkEnvironment foreach {
+ _.getPartitionManager.failIntermediateResultPartitions(executionID)
+ }
+ }
+
+ /**
+ * Handle unmatched messages with an exception.
+ */
+ override def unhandled(message: Any): Unit = {
+ // let the actor crash
+ throw new RuntimeException("Received unknown message " + message)
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManagerProfiler.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManagerProfiler.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManagerProfiler.scala
index 1a7c31d..abd44b1 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManagerProfiler.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManagerProfiler.scala
@@ -137,6 +137,14 @@ ActorLogMessages with ActorLogging {
}
}
+ /**
+ * Handle unmatched messages with an exception.
+ */
+ override def unhandled(message: Any): Unit = {
+ // let the actor crash
+ throw new RuntimeException("Received unknown message " + message)
+ }
+
def startMonitoring(): Unit = {
val interval = new FiniteDuration(reportInterval, TimeUnit.MILLISECONDS)
val delay = new FiniteDuration((reportInterval * Math.random()).toLong, TimeUnit.MILLISECONDS)
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/resources/log4j-test.properties b/flink-runtime/src/test/resources/log4j-test.properties
index 0b686e5..04ec355 100644
--- a/flink-runtime/src/test/resources/log4j-test.properties
+++ b/flink-runtime/src/test/resources/log4j-test.properties
@@ -16,7 +16,7 @@
# limitations under the License.
################################################################################
-# Set root logger level to DEBUG and its only appender to A1.
+# Set root logger level to OFF and its only appender to A1.
log4j.rootLogger=OFF, A1
# A1 is set to be a ConsoleAppender.
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
index d2fe4c2..89d5d43 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerITCase.scala
@@ -72,7 +72,7 @@ WordSpecLike with Matchers with BeforeAndAfterAll {
expectMsg(SubmissionFailure(jobGraph.getJobID, new NoResourceAvailableException(1,1,0)))
- expectNoMsg()
+ expectMsg(JobResultFailed(jobGraph.getJobID, new NoResourceAvailableException(1,1,0)))
}
jm ! NotifyWhenJobRemoved(jobGraph.getJobID)
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java
index 3138bb6..3226651 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java
@@ -28,7 +28,7 @@ import akka.util.Timeout;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.akka.AkkaUtils;
-import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.client.JobCancellationException;
import org.apache.flink.runtime.messages.JobClientMessages;
import org.apache.flink.runtime.messages.JobManagerMessages;
import org.apache.flink.test.util.ForkableFlinkMiniCluster;
@@ -123,12 +123,10 @@ public abstract class CancellingTestBase {
try {
Await.result(result, AkkaUtils.getDefaultTimeout());
- } catch (JobExecutionException exception) {
- if (!exception.isJobCanceledByUser()) {
- throw new IllegalStateException("Job Failed.");
- }
-
+ } catch (JobCancellationException exception) {
jobSuccessfullyCancelled = true;
+ } catch (Exception e) {
+ throw new IllegalStateException("Job failed.", e);
}
if (!jobSuccessfullyCancelled) {
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationClient.scala
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationClient.scala b/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationClient.scala
index dba6344..f7f6967 100644
--- a/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationClient.scala
+++ b/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationClient.scala
@@ -137,4 +137,11 @@ class ApplicationClient extends Actor with ActorLogMessages with ActorLogging {
sender() ! messagesQueue.headOption
}
+ /**
+ * Handle unmatched messages with an exception.
+ */
+ override def unhandled(message: Any): Unit = {
+ // let the actor crash
+ throw new RuntimeException("Received unknown message " + message)
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ac7bf427/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnJobManager.scala
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnJobManager.scala b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnJobManager.scala
index 81bdc3b..2bc81fe 100644
--- a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnJobManager.scala
+++ b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnJobManager.scala
@@ -18,7 +18,7 @@
package org.apache.flink.yarn
-import java.io.{IOException, File}
+import java.io.File
import java.nio.ByteBuffer
import java.util.Collections
@@ -114,7 +114,8 @@ trait YarnJobManager extends ActorLogMessages {
sender() ! new FlinkYarnClusterStatus(instanceManager.getNumberOfRegisteredTaskManagers,
instanceManager.getTotalNumberOfSlots)
- case StartYarnSession(conf, actorSystemPort, webServerPort) => startYarnSession(conf, actorSystemPort, webServerPort)
+ case StartYarnSession(conf, actorSystemPort, webServerPort) =>
+ startYarnSession(conf, actorSystemPort, webServerPort)
case PollContainerCompletion =>
rmClientOption match {
@@ -168,7 +169,9 @@ trait YarnJobManager extends ActorLogMessages {
}
}
- private def startYarnSession(conf: Configuration, actorSystemPort: Int, webServerPort: Int): Unit = {
+ private def startYarnSession(conf: Configuration,
+ actorSystemPort: Int,
+ webServerPort: Int): Unit = {
Try {
log.info("Start yarn session.")
val memoryPerTaskManager = env.get(FlinkYarnClient.ENV_TM_MEMORY).toInt