You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by al...@apache.org on 2019/03/30 12:25:55 UTC

[flink] branch master updated: [FLINK-12015] Fix TaskManagerRunnerTest instability

This is an automated email from the ASF dual-hosted git repository.

aljoscha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new 10bf3b0  [FLINK-12015] Fix TaskManagerRunnerTest instability
10bf3b0 is described below

commit 10bf3b0b754b5eded1f9a18089d2d0c0c053aafb
Author: Aljoscha Krettek <al...@gmail.com>
AuthorDate: Tue Mar 26 17:44:07 2019 +0100

    [FLINK-12015] Fix TaskManagerRunnerTest instability
    
    Before, the was a race condition between the termination future in
    TaskManagerRunner completing and the asynchronous shutdown part here:
    https://github.com/apache/flink/blob/70107c4647ecac3df9b2b8c7920e7cb99ad550f1/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java#L258
    
    The test would go out of the block that was waiting on the future but
    the shutdown code that is executed after the future completes is
    executed asynchronously, so is not guaranteed to have run at that point.
    
    This also refactors the code a bit to make it more obvious what is
    happening and removes the SecurityManagerContext because it was
    obscuring the problem.
    
    This was analyzed by Igal and me, and mostly fixed by Igal.
---
 .../taskexecutor/TaskManagerRunnerTest.java        | 68 +++++++++++-----------
 .../runtime/testutils/SecurityManagerContext.java  | 50 ----------------
 .../SystemExitTrackingSecurityManager.java         | 37 +++++-------
 3 files changed, 48 insertions(+), 107 deletions(-)

diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunnerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunnerTest.java
index 584b120..4110e5f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunnerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunnerTest.java
@@ -22,17 +22,15 @@ import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.JobManagerOptions;
 import org.apache.flink.configuration.TaskManagerOptions;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.testutils.SecurityManagerContext;
 import org.apache.flink.runtime.testutils.SystemExitTrackingSecurityManager;
 import org.apache.flink.util.TestLogger;
-import org.apache.flink.util.function.RunnableWithException;
 
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
 
-import java.util.concurrent.TimeUnit;
-
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertThat;
@@ -45,44 +43,44 @@ public class TaskManagerRunnerTest extends TestLogger {
 	@Rule
 	public final Timeout timeout = Timeout.seconds(30);
 
-	@Test
-	public void testShouldShutdownOnFatalError() throws Exception {
-		try (TaskManagerRunner taskManagerRunner = createTaskManagerRunner(createConfiguration())) {
-			taskManagerRunner.start();
+	private SystemExitTrackingSecurityManager systemExitTrackingSecurityManager;
+	private TaskManagerRunner taskManagerRunner;
 
-			final SystemExitTrackingSecurityManager systemExitTrackingSecurityManager =
-				runWithSystemExitTracking(() -> {
-					taskManagerRunner.onFatalError(new RuntimeException());
-					taskManagerRunner.getTerminationFuture().get(30, TimeUnit.SECONDS);
-				});
+	@Before
+	public void before() {
+		systemExitTrackingSecurityManager = new SystemExitTrackingSecurityManager();
+		System.setSecurityManager(systemExitTrackingSecurityManager);
+	}
 
-			assertThat(systemExitTrackingSecurityManager.getCount(), is(equalTo(1)));
-			assertThat(systemExitTrackingSecurityManager.getStatus(), is(equalTo(TaskManagerRunner.RUNTIME_FAILURE_RETURN_CODE)));
+	@After
+	public void after() throws Exception {
+		System.setSecurityManager(null);
+		if (taskManagerRunner != null) {
+			taskManagerRunner.close();
 		}
 	}
 
 	@Test
-	public void testShouldShutdownIfRegistrationWithJobManagerFails() throws Exception {
-		final Configuration configuration = createConfiguration();
-		configuration.setString(TaskManagerOptions.REGISTRATION_TIMEOUT, "10 ms");
-
-		try (TaskManagerRunner taskManagerRunner = createTaskManagerRunner(configuration)) {
+	public void testShouldShutdownOnFatalError() throws Exception {
+		Configuration configuration = createConfiguration();
+		// very high timeout, to ensure that we don't fail because of registration timeouts
+		configuration.setString(TaskManagerOptions.REGISTRATION_TIMEOUT, "42 h");
+		taskManagerRunner = createTaskManagerRunner(configuration);
 
-			final SystemExitTrackingSecurityManager systemExitTrackingSecurityManager =
-				runWithSystemExitTracking(() -> {
-					taskManagerRunner.start();
-					taskManagerRunner.getTerminationFuture().get();
-				});
+		taskManagerRunner.onFatalError(new RuntimeException());
 
-			assertThat(systemExitTrackingSecurityManager.getCount(), is(equalTo(1)));
-			assertThat(systemExitTrackingSecurityManager.getStatus(), is(equalTo(TaskManagerRunner.RUNTIME_FAILURE_RETURN_CODE)));
-		}
+		Integer statusCode = systemExitTrackingSecurityManager.getSystemExitFuture().get();
+		assertThat(statusCode, is(equalTo(TaskManagerRunner.RUNTIME_FAILURE_RETURN_CODE)));
 	}
 
-	private static SystemExitTrackingSecurityManager runWithSystemExitTracking(final RunnableWithException runnable) {
-		final SystemExitTrackingSecurityManager systemExitTrackingSecurityManager = new SystemExitTrackingSecurityManager();
-		SecurityManagerContext.runWithSecurityManager(systemExitTrackingSecurityManager, runnable);
-		return systemExitTrackingSecurityManager;
+	@Test
+	public void testShouldShutdownIfRegistrationWithJobManagerFails() throws Exception {
+		Configuration configuration = createConfiguration();
+		configuration.setString(TaskManagerOptions.REGISTRATION_TIMEOUT, "10 ms");
+		taskManagerRunner = createTaskManagerRunner(configuration);
+
+		Integer statusCode = systemExitTrackingSecurityManager.getSystemExitFuture().get();
+		assertThat(statusCode, is(equalTo(TaskManagerRunner.RUNTIME_FAILURE_RETURN_CODE)));
 	}
 
 	private static Configuration createConfiguration() {
@@ -93,6 +91,8 @@ public class TaskManagerRunnerTest extends TestLogger {
 	}
 
 	private static TaskManagerRunner createTaskManagerRunner(final Configuration configuration) throws Exception {
-		return new TaskManagerRunner(configuration, ResourceID.generate());
+		TaskManagerRunner taskManagerRunner = new TaskManagerRunner(configuration, ResourceID.generate());
+		taskManagerRunner.start();
+		return taskManagerRunner;
 	}
-}
+}
\ No newline at end of file
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/SecurityManagerContext.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/SecurityManagerContext.java
deleted file mode 100644
index 39c56de..0000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/SecurityManagerContext.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.runtime.testutils;
-
-import org.apache.flink.util.function.RunnableWithException;
-
-/**
- * Enables to run code with a {@link SecurityManager}.
- */
-public class SecurityManagerContext implements AutoCloseable {
-
-	private final SecurityManager previousSecurityManager;
-
-	public SecurityManagerContext(final SecurityManager newSecurityManager) {
-		this.previousSecurityManager = System.getSecurityManager();
-		System.setSecurityManager(newSecurityManager);
-	}
-
-	public static void runWithSecurityManager(final SecurityManager securityManager, final RunnableWithException runnable) {
-		try (SecurityManagerContext ignored = new SecurityManagerContext(securityManager)) {
-			try {
-				runnable.run();
-			} catch (Exception e) {
-				throw new RuntimeException(e);
-			}
-		}
-	}
-
-	@Override
-	public void close() {
-		System.setSecurityManager(previousSecurityManager);
-	}
-}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/SystemExitTrackingSecurityManager.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/SystemExitTrackingSecurityManager.java
index d0f4cc0..f0a82d7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/SystemExitTrackingSecurityManager.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/SystemExitTrackingSecurityManager.java
@@ -20,46 +20,37 @@
 package org.apache.flink.runtime.testutils;
 
 import java.security.Permission;
-
-import static org.apache.flink.util.Preconditions.checkState;
+import java.util.concurrent.CompletableFuture;
 
 /**
- * SecurityManager implementation that forbids and tracks calls to {@link System#exit(int)}.
+ * {@link SecurityManager} implementation blocks calls to {@link System#exit(int)}. On the first
+ * call to {@link System#exit(int)}, we complete the future that can be retrieved via {@link
+ * #getSystemExitFuture()}.
  */
 public class SystemExitTrackingSecurityManager extends SecurityManager {
 
-	private int status;
-
-	private int count;
+	private final CompletableFuture<Integer> systemExitFuture = new CompletableFuture<>();
 
 	@Override
 	public void checkPermission(final Permission perm) {
-
 	}
 
 	@Override
 	public void checkPermission(final Permission perm, final Object context) {
-
 	}
 
 	@Override
 	public synchronized void checkExit(final int status) {
-		this.status = status;
-		++count;
-		throw new SecurityException("SystemExitTrackingSecurityManager is installed. JVM will not exit");
+		systemExitFuture.complete(status);
+		throw new SecurityException(
+				"SystemExitTrackingSecurityManager is installed. JVM will not exit");
 	}
 
-	public synchronized int getStatus() {
-		checkState(isSystemExitCalled());
-		return status;
+	/**
+	 * Returns a {@link CompletableFuture} that is completed with the exit code when {@link
+	 * System#exit(int)} is called.
+	 */
+	public CompletableFuture<Integer> getSystemExitFuture() {
+		return systemExitFuture;
 	}
-
-	public synchronized boolean isSystemExitCalled() {
-		return getCount() > 0;
-	}
-
-	public synchronized int getCount() {
-		return count;
-	}
-
 }