You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ga...@apache.org on 2019/02/05 13:58:22 UTC

[flink] 01/03: [FLINK-11362][tests] Port TaskManagerComponentsStartupShutdownTest to new code base

This is an automated email from the ASF dual-hosted git repository.

gary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit d7c49f5065511f437e37a42361dc2906365b3d82
Author: tison <wa...@gmail.com>
AuthorDate: Fri Feb 1 14:34:29 2019 +0800

    [FLINK-11362][tests] Port TaskManagerComponentsStartupShutdownTest to new code base
    
    This closes #7632.
---
 .../runtime/taskexecutor/TaskExecutorTest.java     |  75 +++++++
 .../TaskManagerComponentsStartupShutdownTest.java  | 219 ---------------------
 2 files changed, 75 insertions(+), 219 deletions(-)

diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
index 2a5fde0..de229b2 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java
@@ -52,6 +52,8 @@ import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices
 import org.apache.flink.runtime.highavailability.nonha.standalone.StandaloneHaServices;
 import org.apache.flink.runtime.instance.HardwareDescription;
 import org.apache.flink.runtime.instance.InstanceID;
+import org.apache.flink.runtime.io.disk.iomanager.IOManager;
+import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync;
 import org.apache.flink.runtime.io.network.NetworkEnvironment;
 import org.apache.flink.runtime.io.network.TaskEventDispatcher;
 import org.apache.flink.runtime.io.network.netty.PartitionProducerStateChecker;
@@ -85,6 +87,7 @@ import org.apache.flink.runtime.taskexecutor.slot.TaskSlotTable;
 import org.apache.flink.runtime.taskexecutor.slot.TimerService;
 import org.apache.flink.runtime.taskmanager.CheckpointResponder;
 import org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation;
+import org.apache.flink.runtime.taskmanager.NetworkEnvironmentConfiguration;
 import org.apache.flink.runtime.taskmanager.Task;
 import org.apache.flink.runtime.taskmanager.TaskManagerActions;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
@@ -232,6 +235,78 @@ public class TaskExecutorTest extends TestLogger {
 	public TestName name = new TestName();
 
 	@Test
+	public void testTaskManagerServicesShutdown() throws Exception {
+		final TaskSlotTable taskSlotTable = new TaskSlotTable(Collections.singleton(ResourceProfile.UNKNOWN), timerService);
+
+		final JobLeaderService jobLeaderService = new JobLeaderService(taskManagerLocation, RetryingRegistrationConfiguration.defaultConfiguration());
+
+		final long heartbeatInterval = 1000L;
+		final long heartbeatTimeout = 1000L;
+
+		final HeartbeatServices heartbeatServices = new HeartbeatServices(heartbeatInterval, heartbeatTimeout);
+
+		final IOManager ioManager = new IOManagerAsync(new String[]{ tmp.newFolder().getAbsolutePath() });
+
+		final TaskExecutorLocalStateStoresManager localStateStoresManager = new TaskExecutorLocalStateStoresManager(
+			false,
+			ioManager.getSpillingDirectories(),
+			Executors.directExecutor());
+
+		final int networkBufNum = 32;
+		final int bufferSize = 32 * 1024;
+		final NetworkEnvironmentConfiguration netConf = new NetworkEnvironmentConfiguration(
+			0.1f,
+			networkBufNum * bufferSize,
+			networkBufNum * bufferSize,
+			bufferSize,
+			IOManager.IOMode.SYNC,
+			0,
+			0,
+			2,
+			8,
+			null);
+		final NetworkEnvironment networkEnvironment = new NetworkEnvironment(
+			32,
+			netConf.networkBufferSize(),
+			netConf.partitionRequestInitialBackoff(),
+			netConf.partitionRequestMaxBackoff(),
+			netConf.networkBuffersPerChannel(),
+			netConf.floatingNetworkBuffersPerGate(),
+			true);
+		networkEnvironment.start();
+
+		final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder()
+			.setTaskManagerLocation(taskManagerLocation)
+			.setIoManager(ioManager)
+			.setNetworkEnvironment(networkEnvironment)
+			.setTaskSlotTable(taskSlotTable)
+			.setJobLeaderService(jobLeaderService)
+			.setTaskStateManager(localStateStoresManager)
+			.build();
+
+		final TaskExecutor taskManager = new TaskExecutor(
+			rpc,
+			taskManagerConfiguration,
+			haServices,
+			taskManagerServices,
+			heartbeatServices,
+			UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(),
+			null,
+			dummyBlobCacheService,
+			testingFatalErrorHandler);
+
+		try {
+			taskManager.start();
+		} finally {
+			RpcUtils.terminateRpcEndpoint(taskManager, timeout);
+		}
+
+		assertThat(taskManagerServices.getMemoryManager().isShutdown(), is(true));
+		assertThat(taskManagerServices.getNetworkEnvironment().isShutdown(), is(true));
+		assertThat(taskManagerServices.getIOManager().isProperlyShutDown(), is(true));
+	}
+
+	@Test
 	public void testHeartbeatTimeoutWithJobManager() throws Exception {
 		final TaskSlotTable taskSlotTable = new TaskSlotTable(Arrays.asList(ResourceProfile.UNKNOWN), timerService);
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
deleted file mode 100644
index 7c46908..0000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerComponentsStartupShutdownTest.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.taskmanager;
-
-import org.apache.flink.api.common.time.Time;
-import org.apache.flink.configuration.AkkaOptions;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.ConfigurationUtils;
-import org.apache.flink.core.memory.MemoryType;
-import org.apache.flink.runtime.akka.AkkaUtils;
-import org.apache.flink.runtime.clusterframework.FlinkResourceManager;
-import org.apache.flink.runtime.clusterframework.standalone.StandaloneResourceManager;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.concurrent.Executors;
-import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders;
-import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
-import org.apache.flink.runtime.highavailability.nonha.embedded.EmbeddedHaServices;
-import org.apache.flink.runtime.io.disk.iomanager.IOManager;
-import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync;
-import org.apache.flink.runtime.io.network.NetworkEnvironment;
-import org.apache.flink.runtime.jobmanager.JobManager;
-import org.apache.flink.runtime.jobmanager.MemoryArchivist;
-import org.apache.flink.runtime.registration.RetryingRegistrationConfiguration;
-import org.apache.flink.runtime.memory.MemoryManager;
-import org.apache.flink.runtime.messages.TaskManagerMessages;
-import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
-import org.apache.flink.runtime.metrics.NoOpMetricRegistry;
-import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup;
-import org.apache.flink.runtime.state.TaskExecutorLocalStateStoresManager;
-import org.apache.flink.runtime.taskexecutor.TaskManagerConfiguration;
-import org.apache.flink.runtime.testingUtils.TestingUtils;
-import org.apache.flink.util.TestLogger;
-
-import akka.actor.ActorRef;
-import akka.actor.ActorSystem;
-import akka.actor.Kill;
-import akka.actor.Props;
-import akka.testkit.JavaTestKit;
-import org.junit.Test;
-
-import java.net.InetAddress;
-import java.util.concurrent.TimeUnit;
-
-import scala.Option;
-import scala.concurrent.Await;
-import scala.concurrent.duration.Duration;
-import scala.concurrent.duration.FiniteDuration;
-
-import static org.junit.Assert.assertTrue;
-
-public class TaskManagerComponentsStartupShutdownTest extends TestLogger {
-
-	/**
-	 * Makes sure that all components are shut down when the TaskManager
-	 * actor is shut down.
-	 */
-	@Test
-	public void testComponentsStartupShutdown() throws Exception {
-
-		final Time timeout = Time.seconds(100);
-		final int BUFFER_SIZE = 32 * 1024;
-
-		Configuration config = new Configuration();
-		config.setString(AkkaOptions.WATCH_HEARTBEAT_INTERVAL, "200 ms");
-		config.setString(AkkaOptions.WATCH_HEARTBEAT_PAUSE, "1 s");
-		config.setInteger(AkkaOptions.WATCH_THRESHOLD, 1);
-
-		final String[] TMP_DIR = ConfigurationUtils.parseTempDirectories(config);
-
-		ActorSystem actorSystem = null;
-
-		HighAvailabilityServices highAvailabilityServices = new EmbeddedHaServices(TestingUtils.defaultExecutor());
-
-		ActorRef jobManager = null;
-		ActorRef taskManager = null;
-
-		try {
-			actorSystem = AkkaUtils.createLocalActorSystem(config);
-
-			jobManager = JobManager.startJobManagerActors(
-				config,
-				actorSystem,
-				TestingUtils.defaultExecutor(),
-				TestingUtils.defaultExecutor(),
-				highAvailabilityServices,
-				NoOpMetricRegistry.INSTANCE,
-				Option.empty(),
-				JobManager.class,
-				MemoryArchivist.class)._1();
-
-			FlinkResourceManager.startResourceManagerActors(
-				config,
-				actorSystem,
-				highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID),
-				StandaloneResourceManager.class);
-
-			final int numberOfSlots = 1;
-
-			// create the components for the TaskManager manually
-			final TaskManagerConfiguration tmConfig = new TaskManagerConfiguration(
-				numberOfSlots,
-				TMP_DIR,
-				timeout,
-				null,
-				Time.milliseconds(500),
-				Time.seconds(30),
-				Time.seconds(10),
-				config,
-				false, // exit-jvm-on-fatal-error
-				FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
-				new String[0],
-				null,
-				null,
-				RetryingRegistrationConfiguration.fromConfiguration(config));
-
-			final int networkBufNum = 32;
-			// note: the network buffer memory configured here is not actually used below but set
-			// accordingly to be consistent
-			final NetworkEnvironmentConfiguration netConf = new NetworkEnvironmentConfiguration(
-					0.1f, networkBufNum * BUFFER_SIZE, networkBufNum * BUFFER_SIZE, BUFFER_SIZE, IOManager.IOMode.SYNC,
-					0, 0, 2, 8, null);
-
-			ResourceID taskManagerId = ResourceID.generate();
-			
-			final TaskManagerLocation connectionInfo = new TaskManagerLocation(taskManagerId, InetAddress.getLocalHost(), 10000);
-
-			final MemoryManager memManager = new MemoryManager(networkBufNum * BUFFER_SIZE, 1, BUFFER_SIZE, MemoryType.HEAP, false);
-			final IOManager ioManager = new IOManagerAsync(TMP_DIR);
-			final NetworkEnvironment network = new NetworkEnvironment(
-				32,
-				netConf.networkBufferSize(),
-				netConf.partitionRequestInitialBackoff(),
-				netConf.partitionRequestMaxBackoff(),
-				netConf.networkBuffersPerChannel(),
-				netConf.floatingNetworkBuffersPerGate(),
-				true);
-
-			network.start();
-
-			TaskExecutorLocalStateStoresManager storesManager = new TaskExecutorLocalStateStoresManager(
-				false,
-				ioManager.getSpillingDirectories(),
-				Executors.directExecutor());
-
-			MetricRegistryConfiguration metricRegistryConfiguration = MetricRegistryConfiguration.fromConfiguration(config);
-
-			// create the task manager
-			final Props tmProps = Props.create(
-				TaskManager.class,
-				tmConfig,
-				taskManagerId,
-				connectionInfo,
-				memManager,
-				ioManager,
-				network,
-				storesManager,
-				numberOfSlots,
-				highAvailabilityServices,
-				new TaskManagerMetricGroup(NoOpMetricRegistry.INSTANCE, connectionInfo.getHostname(), connectionInfo.getResourceID().getResourceIdString()));
-
-			taskManager = actorSystem.actorOf(tmProps);
-
-			final ActorRef finalTaskManager = taskManager;
-
-			new JavaTestKit(actorSystem) {{
-
-				// wait for the TaskManager to be registered
-				new Within(new FiniteDuration(5000L, TimeUnit.SECONDS)) {
-					@Override
-					protected void run() {
-						finalTaskManager.tell(TaskManagerMessages.getNotifyWhenRegisteredAtJobManagerMessage(),
-								getTestActor());
-
-						expectMsgClass(TaskManagerMessages.RegisteredAtJobManager.class);
-					}
-				};
-			}};
-
-			// shut down all actors and the actor system
-			// Kill the Task down the JobManager
-			taskManager.tell(Kill.getInstance(), ActorRef.noSender());
-			jobManager.tell(Kill.getInstance(), ActorRef.noSender());
-
-			// shut down the actors and the actor system
-			actorSystem.terminate();
-			Await.ready(actorSystem.whenTerminated(), Duration.Inf());
-			actorSystem = null;
-
-			// now that the TaskManager is shut down, the components should be shut down as well
-			assertTrue(network.isShutdown());
-			assertTrue(ioManager.isProperlyShutDown());
-			assertTrue(memManager.isShutdown());
-		} finally {
-			if (actorSystem != null) {
-				actorSystem.terminate();
-
-				Await.ready(actorSystem.whenTerminated(), TestingUtils.TESTING_TIMEOUT());
-			}
-
-			highAvailabilityServices.closeAndCleanupAllData();
-		}
-	}
-}