You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2015/02/24 13:31:13 UTC
[6/6] flink git commit: [tests] Add process reaping test for
TaskManager, improves process reaping test for JobManager.
[tests] Add process reaping test for TaskManager, improves process reaping test for JobManager.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1230bcaa
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1230bcaa
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1230bcaa
Branch: refs/heads/master
Commit: 1230bcaa0f531f6260f291dd066f64fe52cc6708
Parents: 70df028
Author: Stephan Ewen <se...@apache.org>
Authored: Tue Feb 24 12:23:59 2015 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Tue Feb 24 12:23:59 2015 +0100
----------------------------------------------------------------------
.../flink/runtime/jobmanager/JobManager.scala | 94 ++++----
.../flink/runtime/taskmanager/TaskManager.scala | 68 +++++-
.../JobManagerProcessReapingTest.java | 83 ++++++-
.../TaskManagerProcessReapingTest.java | 236 +++++++++++++++++++
.../runtime/testutils/CommonTestUtils.java | 24 ++
5 files changed, 447 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1230bcaa/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index a1642b4..2671f2d 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -94,17 +94,18 @@ class JobManager(val configuration: Configuration,
val profiler: Option[ActorRef],
val defaultExecutionRetries: Int,
val delayBetweenRetries: Long,
- implicit val timeout: FiniteDuration)
+ val timeout: FiniteDuration)
extends Actor with ActorLogMessages with ActorLogging {
- import context._
-
- val LOG = JobManager.LOG
-
- // List of current jobs running
- val currentJobs = scala.collection.mutable.HashMap[JobID, (ExecutionGraph, JobInfo)]()
+ /** Reference to the log, for debugging */
+ protected val LOG = JobManager.LOG
+ /** List of current jobs running jobs */
+ protected val currentJobs = scala.collection.mutable.HashMap[JobID, (ExecutionGraph, JobInfo)]()
+ /**
+ * Run when the job manager is started. Simply logs an informational message.
+ */
override def preStart(): Unit = {
LOG.info(s"Starting JobManager at ${self.path}.")
}
@@ -138,6 +139,11 @@ class JobManager(val configuration: Configuration,
}
}
+ /**
+ * Central work method of the JobManager actor. Receives messages and reacts to them.
+ *
+ * @return
+ */
override def receiveWithLogMessages: Receive = {
case RegisterTaskManager(connectionInfo, hardwareInformation, numberOfSlots) =>
@@ -182,7 +188,7 @@ class JobManager(val configuration: Configuration,
// execute the cancellation asynchronously
Future {
executionGraph.cancel()
- }
+ }(context.dispatcher)
sender ! CancellationSuccess(jobID)
case None =>
@@ -198,10 +204,12 @@ class JobManager(val configuration: Configuration,
currentJobs.get(taskExecutionState.getJobID) match {
case Some((executionGraph, _)) =>
val originalSender = sender
+
Future {
val result = executionGraph.updateState(taskExecutionState)
originalSender ! result
- }
+ }(context.dispatcher)
+
sender ! true
case None => log.error("Cannot find execution graph for ID {} to change state to {}.",
taskExecutionState.getJobID, taskExecutionState.getExecutionState)
@@ -603,6 +611,7 @@ object JobManager {
EnvironmentInformation.logEnvironmentInfo(LOG, "JobManager")
checkJavaVersion()
+ // parsing the command line arguments
val (configuration: Configuration,
executionMode: ExecutionMode,
listeningHost: String, listeningPort: Int) =
@@ -617,16 +626,17 @@ object JobManager {
}
}
- // we may want to check that the JobManager hostname is in the config
+ // we want to check that the JobManager hostname is in the config
// if it is not in there, the actor system will bind to the loopback interface's
// address and will not be reachable from anyone remote
if (listeningHost == null) {
val message = "Config parameter '" + ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY +
- "' is missing (hostname or address to bind JobManager to)."
+ "' is missing (hostname/address to bind JobManager to)."
LOG.error(message)
System.exit(STARTUP_FAILURE_RETURN_CODE)
}
+ // run the job manager
try {
if (SecurityUtils.isSecurityEnabled) {
LOG.info("Security is enabled. Starting secure JobManager.")
@@ -647,13 +657,28 @@ object JobManager {
}
}
-
+ /**
+ * Starts and runs the JobManager with all its components. First, this method starts a
+ * dedicated actor system for the JobManager. Second, its starts all components of the
+ * JobManager (including library cache, instance manager, scheduler). Finally, it starts
+ * the JobManager actor itself.
+ *
+ * This method blocks indefinitely (or until the JobManager's actor system is shut down).
+ *
+ * @param configuration The configuration object for the JobManager.
+ * @param executionMode The execution mode in which to run. Execution mode LOCAL will spawn an
+ * an additional TaskManager in the same process.
+ * @param listeningAddress The hostname where the JobManager should listen for messages.
+ * @param listeningPort The port where the JobManager should listen for messages.
+ */
def runJobManager(configuration: Configuration,
executionMode: ExecutionMode,
listeningAddress: String,
listeningPort: Int) : Unit = {
LOG.info("Starting JobManager")
+
+ // Bring up the job manager actor system first, bind it to the given address.
LOG.debug("Starting JobManager actor system")
val jobManagerSystem = try {
@@ -674,18 +699,18 @@ object JobManager {
}
try {
- LOG.debug("Starting JobManager actor")
-
// bring up the job manager actor
+ LOG.debug("Starting JobManager actor")
val (jobManager, archiver) = startJobManagerActors(configuration, jobManagerSystem)
- // start a process reaper that watches the JobManager
+ // start a process reaper that watches the JobManager. If the JobManager actor dies,
+ // the process reaper will kill the JVM process (to ensure easy failure detection)
jobManagerSystem.actorOf(
Props(classOf[ProcessReaper], jobManager, LOG, RUNTIME_FAILURE_RETURN_CODE),
"JobManager_Process_Reaper")
// bring up a local task manager, if needed
- if(executionMode.equals(LOCAL)){
+ if (executionMode.equals(LOCAL)) {
LOG.info("Starting embedded TaskManager for JobManager's LOCAL mode execution")
TaskManager.startActorWithConfiguration("", TaskManager.TASK_MANAGER_NAME, configuration,
@@ -757,15 +782,22 @@ object JobManager {
}
/**
- * Extracts the job manager configuration values from a configuration instance.
+ * Create the job manager components as (instanceManager, scheduler, libraryCacheManager,
+ * archiverProps, accumulatorManager, profiler, defaultExecutionRetries,
+ * delayBetweenRetries, timeout)
*
- * @param configuration Object with the user provided configuration values
- * @return Tuple of (number of archived jobs, profiling enabled, cleanup interval of the library
- * cache manager, default number of execution retries, delay between retries)
+ * @param configuration The configuration from which to parse the config values.
+ * @return The members for a default JobManager.
*/
- def parseConfiguration(configuration: Configuration): (Int, Boolean, Long, Int, Long) = {
+ def createJobManagerComponents(configuration: Configuration) :
+ (InstanceManager, FlinkScheduler, BlobLibraryCacheManager,
+ Props, AccumulatorManager, Option[Props], Int, Long, FiniteDuration, Int) = {
+
+ val timeout: FiniteDuration = AkkaUtils.getTimeout(configuration)
+
val archiveCount = configuration.getInteger(ConfigConstants.JOB_MANAGER_WEB_ARCHIVE_COUNT,
ConfigConstants.DEFAULT_JOB_MANAGER_WEB_ARCHIVE_COUNT)
+
val profilingEnabled = configuration.getBoolean(ProfilingUtils.PROFILE_JOB_KEY, false)
val cleanupInterval = configuration.getLong(
@@ -780,26 +812,6 @@ object JobManager {
ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE,
ConfigConstants.DEFAULT_AKKA_ASK_TIMEOUT)).toMillis
- (archiveCount, profilingEnabled, cleanupInterval, executionRetries, delayBetweenRetries)
- }
-
- /**
- * Create the job manager members as (instanceManager, scheduler, libraryCacheManager,
- * archiverProps, accumulatorManager, profiler, defaultExecutionRetries,
- * delayBetweenRetries, timeout)
- *
- * @param configuration The configuration from which to parse the config values.
- * @return The members for a default JobManager.
- */
- def createJobManagerComponents(configuration: Configuration) :
- (InstanceManager, FlinkScheduler, BlobLibraryCacheManager,
- Props, AccumulatorManager, Option[Props], Int, Long, FiniteDuration, Int) = {
-
- val timeout: FiniteDuration = AkkaUtils.getTimeout(configuration)
-
- val (archiveCount, profilingEnabled, cleanupInterval, executionRetries, delayBetweenRetries) =
- parseConfiguration(configuration)
-
val archiveProps: Props = Props(classOf[MemoryArchivist], archiveCount)
val profilerProps: Option[Props] = if (profilingEnabled) {
http://git-wip-us.apache.org/repos/asf/flink/blob/1230bcaa/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 9095eae..52a16bd 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -53,6 +53,7 @@ import org.apache.flink.runtime.messages.TaskManagerMessages._
import org.apache.flink.runtime.messages.TaskManagerProfilerMessages
.{UnregisterProfilingListener, UnmonitorTask, MonitorTask, RegisterProfilingListener}
import org.apache.flink.runtime.net.NetUtils
+import org.apache.flink.runtime.process.ProcessReaper
import org.apache.flink.runtime.profiling.ProfilingUtils
import org.apache.flink.runtime.security.SecurityUtils
import org.apache.flink.runtime.security.SecurityUtils.FlinkSecuredRunner
@@ -60,7 +61,7 @@ import org.apache.flink.runtime.util.EnvironmentInformation
import org.apache.flink.util.ExceptionUtils
import org.slf4j.LoggerFactory
-import scala.concurrent.Future
+import scala.concurrent._
import scala.concurrent.duration._
import scala.util.{Failure, Success}
import scala.language.postfixOps
@@ -721,7 +722,9 @@ import scala.collection.JavaConverters._
object TaskManager {
val LOG = LoggerFactory.getLogger(classOf[TaskManager])
- val FAILURE_RETURN_CODE = -1
+
+ val STARTUP_FAILURE_RETURN_CODE = 1
+ val RUNTIME_FAILURE_RETURN_CODE = 2
val TASK_MANAGER_NAME = "taskmanager"
val PROFILER_NAME = "profiler"
@@ -750,9 +753,16 @@ object TaskManager {
def startActor(hostname: String, port: Int, configuration: Configuration,
taskManagerName: String) : Unit = {
- val (taskManagerSystem, _) = startActorSystemAndActor(hostname, port, configuration,
+ val (taskManagerSystem, taskManager) = startActorSystemAndActor(hostname, port, configuration,
taskManagerName, localAkkaCommunication = false, localTaskManagerCommunication = false)
+ // start a process reaper that watches the JobManager. If the JobManager actor dies,
+ // the process reaper will kill the JVM process (to ensure easy failure detection)
+ taskManagerSystem.actorOf(
+ Props(classOf[ProcessReaper], taskManager, LOG, RUNTIME_FAILURE_RETURN_CODE),
+ "TaskManager_Process_Reaper")
+
+ // block until everything is done
taskManagerSystem.awaitTermination()
}
@@ -804,7 +814,7 @@ object TaskManager {
} getOrElse {
LOG.error(s"TaskManager parseArgs called with ${args.mkString(" ")}.")
LOG.error("CLI parsing failed. Usage: " + parser.usage)
- sys.exit(FAILURE_RETURN_CODE)
+ sys.exit(STARTUP_FAILURE_RETURN_CODE)
}
}
@@ -979,7 +989,43 @@ object TaskManager {
system.actorOf(Props(classOf[TaskManagerProfiler], instancePath, reportInterval), PROFILER_NAME)
}
- def checkTempDirs(tmpDirs: Array[String]): Unit = {
+ // --------------------------------------------------------------------------
+ // Resolving the TaskManager actor
+ // --------------------------------------------------------------------------
+
+ /**
+ * Resolves the TaskManager actor reference in a blocking fashion.
+ *
+ * @param taskManagerUrl The akka URL of the JobManager.
+ * @param system The local actor system that should perform the lookup.
+ * @param timeout The maximum time to wait until the lookup fails.
+ * @throws java.io.IOException Thrown, if the lookup fails.
+ * @return The ActorRef to the TaskManager
+ */
+ @throws(classOf[IOException])
+ def getTaskManagerRemoteReference(taskManagerUrl: String,
+ system: ActorSystem,
+ timeout: FiniteDuration): ActorRef = {
+ try {
+ val future = AkkaUtils.getReference(taskManagerUrl, system, timeout)
+ Await.result(future, timeout)
+ }
+ catch {
+ case e @ (_ : ActorNotFound | _ : TimeoutException) =>
+ throw new IOException(
+ s"TaskManager at $taskManagerUrl not reachable. " +
+ s"Please make sure that the TaskManager is running and its port is reachable.", e)
+
+ case e: IOException =>
+ throw new IOException("Could not connect to TaskManager at " + taskManagerUrl, e)
+ }
+ }
+
+ // --------------------------------------------------------------------------
+ // Miscellaneous Utilities
+ // --------------------------------------------------------------------------
+
+ private def checkTempDirs(tmpDirs: Array[String]): Unit = {
tmpDirs.zipWithIndex.foreach {
case (dir: String, _) =>
val file = new File(dir)
@@ -1010,6 +1056,12 @@ object TaskManager {
}
}
+ /**
+ * Gets the memory footprint of the JVM in a string representation.
+ *
+ * @param memoryMXBean The memory management bean used to access the memory statistics.
+ * @return A string describing how much heap memory and direct memory are allocated and used.
+ */
private def getMemoryUsageStatsAsString(memoryMXBean: MemoryMXBean): String = {
val heap = memoryMXBean.getHeapMemoryUsage
val nonHeap = memoryMXBean.getNonHeapMemoryUsage
@@ -1026,6 +1078,12 @@ object TaskManager {
s"NON HEAP: $nonHeapUsed/$nonHeapCommitted/$nonHeapMax MB (used/committed/max)]"
}
+ /**
+ * Gets the garbage collection statistics from the JVM.
+ *
+ * @param gcMXBeans The collection of garbage collector beans.
+ * @return A string denoting the number of times and total elapsed time in garbage collection.
+ */
private def getGarbageCollectorStatsAsString(gcMXBeans: Iterable[GarbageCollectorMXBean])
: String = {
val beans = gcMXBeans map {
http://git-wip-us.apache.org/repos/asf/flink/blob/1230bcaa/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerProcessReapingTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerProcessReapingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerProcessReapingTest.java
index 74ed938..ae3a771 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerProcessReapingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerProcessReapingTest.java
@@ -27,14 +27,19 @@ import akka.actor.ActorRef;
import akka.actor.ActorSystem;
import akka.actor.PoisonPill;
import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.net.NetUtils;
+import org.apache.flink.runtime.testutils.CommonTestUtils;
import org.junit.Test;
-import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import scala.Some;
import scala.Tuple2;
import scala.concurrent.duration.FiniteDuration;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
import java.net.InetSocketAddress;
import java.util.concurrent.TimeUnit;
@@ -43,33 +48,45 @@ import java.util.concurrent.TimeUnit;
*/
public class JobManagerProcessReapingTest {
- private static final int JOB_MANAGER_PORT = 56532;
-
@Test
public void testReapProcessOnFailure() {
Process jmProcess = null;
ActorSystem localSystem = null;
+ final StringWriter processOutput = new StringWriter();
+
try {
String javaCommand = getJavaCommandPath();
// check that we run this test only if the java command
// is available on this machine
if (javaCommand == null) {
+ System.out.println("---- Skipping JobManagerProcessReapingTest : Could not find java executable ----");
return;
}
+ // create a logging file for the process
+ File tempLogFile = File.createTempFile("testlogconfig", "properties");
+ tempLogFile.deleteOnExit();
+ CommonTestUtils.printLog4jDebugConfig(tempLogFile);
+
+ int jobManagerPort = NetUtils.getAvailablePort();
+
// start a JobManger process
String[] command = new String[] {
javaCommand,
- "-Dlog.level=OFF",
+ "-Dlog.level=DEBUG",
+ "-Dlog4j.configuration=file:" + tempLogFile.getAbsolutePath(),
"-Xms256m", "-Xmx256m",
"-classpath", getCurrentClasspath(),
- JobManagerTestEntryPoint.class.getName()
+ JobManagerTestEntryPoint.class.getName(),
+ String.valueOf(jobManagerPort)
};
+ // spawn the process and collect its output
ProcessBuilder bld = new ProcessBuilder(command);
jmProcess = bld.start();
+ new PipeForwarder(jmProcess.getErrorStream(), processOutput);
// start another actor system so we can send something to the JobManager
Tuple2<String, Object> localAddress = new Tuple2<String, Object>("localhost", 0);
@@ -82,8 +99,8 @@ public class JobManagerProcessReapingTest {
for (int i = 0; i < 20; i++) {
try {
jobManagerRef = JobManager.getJobManagerRemoteReference(
- new InetSocketAddress("localhost", JOB_MANAGER_PORT),
- localSystem, new FiniteDuration(20, TimeUnit.SECONDS));
+ new InetSocketAddress("localhost", jobManagerPort),
+ localSystem, new FiniteDuration(5, TimeUnit.SECONDS));
break;
}
catch (Throwable t) {
@@ -117,8 +134,14 @@ public class JobManagerProcessReapingTest {
}
catch (Exception e) {
e.printStackTrace();
+ printProcessLog(processOutput.toString());
fail(e.getMessage());
}
+ catch (Error e) {
+ e.printStackTrace();
+ printProcessLog(processOutput.toString());
+ throw e;
+ }
finally {
if (jmProcess != null) {
jmProcess.destroy();
@@ -129,17 +152,24 @@ public class JobManagerProcessReapingTest {
}
}
+ private static void printProcessLog(String log) {
+ System.out.println("-----------------------------------------");
+ System.out.println(" BEGIN SPAWNED PROCESS LOG");
+ System.out.println("-----------------------------------------");
+ System.out.println(log);
+ System.out.println("-----------------------------------------");
+ System.out.println(" END SPAWNED PROCESS LOG");
+ System.out.println("-----------------------------------------");
+ }
+
// --------------------------------------------------------------------------------------------
public static class JobManagerTestEntryPoint {
public static void main(String[] args) {
try {
- Configuration cfg = new Configuration();
- cfg.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost");
- cfg.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, JOB_MANAGER_PORT);
-
- JobManager.runJobManager(cfg, ExecutionMode.CLUSTER(), "localhost", JOB_MANAGER_PORT);
+ int port = Integer.parseInt(args[0]);
+ JobManager.runJobManager(new Configuration(), ExecutionMode.CLUSTER(), "localhost", port);
System.exit(0);
}
catch (Throwable t) {
@@ -147,4 +177,33 @@ public class JobManagerProcessReapingTest {
}
}
}
+
+ private static class PipeForwarder extends Thread {
+
+ private final StringWriter target;
+ private final InputStream source;
+
+ public PipeForwarder(InputStream source, StringWriter target) {
+ super("Pipe Forwarder");
+ setDaemon(true);
+
+ this.source = source;
+ this.target = target;
+
+ start();
+ }
+
+ @Override
+ public void run() {
+ try {
+ int next;
+ while ((next = source.read()) != -1) {
+ target.write(next);
+ }
+ }
+ catch (IOException e) {
+ // terminate
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1230bcaa/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerProcessReapingTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerProcessReapingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerProcessReapingTest.java
new file mode 100644
index 0000000..b92cfd4
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerProcessReapingTest.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskmanager;
+
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.actor.PoisonPill;
+
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.akka.AkkaUtils;
+
+import org.apache.flink.runtime.jobmanager.JobManager;
+import org.apache.flink.runtime.net.NetUtils;
+import org.apache.flink.runtime.testutils.CommonTestUtils;
+import org.junit.Test;
+
+import scala.Some;
+import scala.Tuple2;
+import scala.concurrent.duration.FiniteDuration;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import static org.apache.flink.runtime.testutils.CommonTestUtils.getCurrentClasspath;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.getJavaCommandPath;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.isProcessAlive;
+
+/**
+ * Tests that the JobManager process properly exits when the JobManager actor dies.
+ */
+public class TaskManagerProcessReapingTest {
+
+ private static final String TASK_MANAGER_ACTOR_NAME = "TEST_TM";
+
+ @Test
+ public void testReapProcessOnFailure() {
+ Process taskManagerProcess = null;
+ ActorSystem jmActorSystem = null;
+
+ final StringWriter processOutput = new StringWriter();
+
+ try {
+ String javaCommand = getJavaCommandPath();
+
+ // check that we run this test only if the java command
+ // is available on this machine
+ if (javaCommand == null) {
+ System.out.println("---- Skipping TaskManagerProcessReapingTest : Could not find java executable ----");
+ return;
+ }
+
+ // create a logging file for the process
+ File tempLogFile = File.createTempFile("testlogconfig", "properties");
+ tempLogFile.deleteOnExit();
+ CommonTestUtils.printLog4jDebugConfig(tempLogFile);
+
+ final int jobManagerPort = NetUtils.getAvailablePort();
+
+ // start a JobManager
+ Tuple2<String, Object> localAddress = new Tuple2<String, Object>("localhost", jobManagerPort);
+ jmActorSystem = AkkaUtils.createActorSystem(
+ new Configuration(), new Some<Tuple2<String, Object>>(localAddress));
+
+ JobManager.startJobManagerActors(new Configuration(), jmActorSystem);
+
+ final int taskManagerPort = NetUtils.getAvailablePort();
+
+ // start the task manager process
+ String[] command = new String[] {
+ javaCommand,
+ "-Dlog.level=DEBUG",
+ "-Dlog4j.configuration=file:" + tempLogFile.getAbsolutePath(),
+ "-Xms256m", "-Xmx256m",
+ "-classpath", getCurrentClasspath(),
+ TaskManagerTestEntryPoint.class.getName(),
+ String.valueOf(jobManagerPort), String.valueOf(taskManagerPort)
+ };
+
+ ProcessBuilder bld = new ProcessBuilder(command);
+ taskManagerProcess = bld.start();
+ new PipeForwarder(taskManagerProcess.getErrorStream(), processOutput);
+
+ // grab the reference to the TaskManager. try multiple times, until the process
+ // is started and the TaskManager is up
+ String taskManagerActorName = String.format("akka.tcp://flink@%s:%d/user/%s",
+ "127.0.0.1", taskManagerPort, TASK_MANAGER_ACTOR_NAME);
+
+ ActorRef taskManagerRef = null;
+ for (int i = 0; i < 20; i++) {
+ try {
+ taskManagerRef = TaskManager.getTaskManagerRemoteReference(
+ taskManagerActorName, jmActorSystem, new FiniteDuration(5, TimeUnit.SECONDS));
+ break;
+ }
+ catch (Throwable t) {
+ // TaskManager probably not ready yet
+ }
+ Thread.sleep(500);
+ }
+
+ assertTrue("TaskManager process died", isProcessAlive(taskManagerProcess));
+ assertTrue("TaskManager process did not launch the TaskManager properly", taskManagerRef != null);
+
+ // kill the TaskManager actor
+ taskManagerRef.tell(PoisonPill.getInstance(), ActorRef.noSender());
+
+ // wait for max 5 seconds for the process to terminate
+ {
+ long now = System.currentTimeMillis();
+ long deadline = now + 5000;
+
+ while (now < deadline && isProcessAlive(taskManagerProcess)) {
+ Thread.sleep(100);
+ now = System.currentTimeMillis();
+ }
+ }
+
+ assertFalse("TaskManager process did not terminate upon actor death", isProcessAlive(taskManagerProcess));
+
+ int returnCode = taskManagerProcess.exitValue();
+ assertEquals("TaskManager died, but not because of the process reaper",
+ TaskManager.RUNTIME_FAILURE_RETURN_CODE(), returnCode);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ printProcessLog(processOutput.toString());
+ fail(e.getMessage());
+ }
+ catch (Error e) {
+ e.printStackTrace();
+ printProcessLog(processOutput.toString());
+ throw e;
+ }
+ finally {
+ if (taskManagerProcess != null) {
+ taskManagerProcess.destroy();
+ }
+ if (jmActorSystem != null) {
+ jmActorSystem.shutdown();
+ }
+ }
+ }
+
+ private static void printProcessLog(String log) {
+ System.out.println("-----------------------------------------");
+ System.out.println(" BEGIN SPAWNED PROCESS LOG");
+ System.out.println("-----------------------------------------");
+ System.out.println(log);
+ System.out.println("-----------------------------------------");
+ System.out.println(" END SPAWNED PROCESS LOG");
+ System.out.println("-----------------------------------------");
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public static class TaskManagerTestEntryPoint {
+
+ public static void main(String[] args) {
+ try {
+ int jobManagerPort = Integer.parseInt(args[0]);
+ int taskManagerPort = Integer.parseInt(args[1]);
+
+ Configuration cfg = new Configuration();
+ cfg.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost");
+ cfg.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, jobManagerPort);
+ cfg.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 4);
+ cfg.setInteger(ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY, 100);
+
+ TaskManager.startActor("localhost", taskManagerPort, cfg, TASK_MANAGER_ACTOR_NAME);
+
+ // wait forever
+ Object lock = new Object();
+ synchronized (lock) {
+ lock.wait();
+ }
+ }
+ catch (Throwable t) {
+ System.exit(1);
+ }
+ }
+ }
+
+ private static class PipeForwarder extends Thread {
+
+ private final StringWriter target;
+ private final InputStream source;
+
+ public PipeForwarder(InputStream source, StringWriter target) {
+ super("Pipe Forwarder");
+ setDaemon(true);
+
+ this.source = source;
+ this.target = target;
+
+ start();
+ }
+
+ @Override
+ public void run() {
+ try {
+ int next;
+ while ((next = source.read()) != -1) {
+ target.write(next);
+ }
+ }
+ catch (IOException e) {
+ // terminate
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1230bcaa/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
index fc52417..ca05416 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
@@ -20,14 +20,18 @@ package org.apache.flink.runtime.testutils;
import static org.junit.Assert.fail;
+import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
+import java.io.PrintWriter;
import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
@@ -204,4 +208,24 @@ public class CommonTestUtils {
return true;
}
}
+
+ public static void printLog4jDebugConfig(File file) throws IOException {
+ FileWriter fw = new FileWriter(file);
+ try {
+ PrintWriter writer = new PrintWriter(fw);
+
+ writer.println("log4j.rootLogger=DEBUG, console");
+ writer.println("log4j.appender.console=org.apache.log4j.ConsoleAppender");
+ writer.println("log4j.appender.console.target = System.err");
+ writer.println("log4j.appender.console.layout=org.apache.log4j.PatternLayout");
+ writer.println("log4j.appender.console.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n");
+ writer.println("log4j.logger.org.eclipse.jetty.util.log=OFF");
+
+ writer.flush();
+ writer.close();
+ }
+ finally {
+ fw.close();
+ }
+ }
}