You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by GitBox <gi...@apache.org> on 2020/10/06 13:38:04 UTC

[GitHub] [flink] tillrohrmann commented on a change in pull request #13311: [FLINK-18721] Migrate YarnResourceManager to the new YarnResourceManagerDriver

tillrohrmann commented on a change in pull request #13311:
URL: https://github.com/apache/flink/pull/13311#discussion_r500102360



##########
File path: flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnResourceManagerConfiguration.java
##########
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn.configuration;
+
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.yarn.YarnConfigKeys;
+
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+
+import javax.annotation.Nullable;
+
+import java.util.Map;
+
+/**
+ * Configuration specific to {@link org.apache.flink.yarn.YarnResourceManager}.
+ */
+public class YarnResourceManagerConfiguration {
+	@Nullable private final String webInterfaceUrl;
+	private final String rpcAddress;
+	private final String yarnFiles;
+	private final String flinkClasspath;
+	private final String clientShipFiles;
+	private final String flinkDistJar;
+	private final String currentDir;
+	@Nullable private final String remoteKeytabPath;
+	@Nullable private final String localKeytabPath;
+	@Nullable private final String keytabPrinciple;
+	@Nullable private final String krb5Path;
+	@Nullable private final String yarnSiteXMLPath;
+
+	public YarnResourceManagerConfiguration(
+		Map<String, String> env,
+		@Nullable String webInterfaceUrl,
+		String rpcAddress) {

Review comment:
       nit: I would change the order of these two parameters.

##########
File path: flink-yarn/src/main/java/org/apache/flink/yarn/YarnNodeManagerClientFactory.java
##########
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+
+/**
+ * Factory interface for {@link NMClientAsync}.
+ */
+public interface YarnNodeManagerClientFactory {
+	NMClientAsync createNodeManagerClient(NMClientAsync.CallbackHandler callbackHandler);

Review comment:
       nit: JavaDocs are missing

##########
File path: flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManagerDriver.java
##########
@@ -0,0 +1,609 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.configuration.TaskManagerOptionsInternal;
+import org.apache.flink.runtime.clusterframework.ApplicationStatus;
+import org.apache.flink.runtime.clusterframework.BootstrapTools;
+import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters;
+import org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.FutureUtils;
+import org.apache.flink.runtime.resourcemanager.active.AbstractResourceManagerDriver;
+import org.apache.flink.runtime.resourcemanager.active.ResourceManagerDriver;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
+import org.apache.flink.runtime.webmonitor.history.HistoryServerUtils;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.yarn.configuration.YarnConfigOptions;
+import org.apache.flink.yarn.configuration.YarnConfigOptionsInternal;
+import org.apache.flink.yarn.configuration.YarnResourceManagerConfiguration;
+
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+/**
+ * Implementation of {@link ResourceManagerDriver} for Yarn deployment.
+ */
+public class YarnResourceManagerDriver extends AbstractResourceManagerDriver<YarnWorkerNode> {
+
+	private static final Priority RM_REQUEST_PRIORITY = Priority.newInstance(1);
+
+	/** Environment variable name of the hostname given by the YARN.

Review comment:
       ```suggestion
   	/** Environment variable name of the hostname given by YARN.
   ```

##########
File path: flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManagerDriver.java
##########
@@ -0,0 +1,609 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.configuration.TaskManagerOptionsInternal;
+import org.apache.flink.runtime.clusterframework.ApplicationStatus;
+import org.apache.flink.runtime.clusterframework.BootstrapTools;
+import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters;
+import org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.FutureUtils;
+import org.apache.flink.runtime.resourcemanager.active.AbstractResourceManagerDriver;
+import org.apache.flink.runtime.resourcemanager.active.ResourceManagerDriver;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
+import org.apache.flink.runtime.webmonitor.history.HistoryServerUtils;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.yarn.configuration.YarnConfigOptions;
+import org.apache.flink.yarn.configuration.YarnConfigOptionsInternal;
+import org.apache.flink.yarn.configuration.YarnResourceManagerConfiguration;
+
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+/**
+ * Implementation of {@link ResourceManagerDriver} for Yarn deployment.
+ */
+public class YarnResourceManagerDriver extends AbstractResourceManagerDriver<YarnWorkerNode> {
+
+	private static final Priority RM_REQUEST_PRIORITY = Priority.newInstance(1);
+
+	/** Environment variable name of the hostname given by the YARN.
+	 * In task executor we use the hostnames given by YARN consistently throughout akka */
+	private static final String ENV_FLINK_NODE_ID = "_FLINK_NODE_ID";
+
+	static final String ERROR_MESSAGE_ON_SHUTDOWN_REQUEST = "Received shutdown request from YARN ResourceManager.";
+
+	private final YarnConfiguration yarnConfig;
+
+	/** The process environment variables. */
+	private final YarnResourceManagerConfiguration configuration;
+
+	/** Default heartbeat interval between this resource manager and the YARN ResourceManager. */
+	private final int yarnHeartbeatIntervalMillis;
+
+	/** Client to communicate with the Resource Manager (YARN's master). */
+	private AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient;
+
+	/** The heartbeat interval while the resource master is waiting for containers. */
+	private final int containerRequestHeartbeatIntervalMillis;
+
+	/** Client to communicate with the Node manager and launch TaskExecutor processes. */
+	private NMClientAsync nodeManagerClient;
+
+	/** Request resource futures, keyed by container ids. */

Review comment:
       Comment seems to be wrong. Isn't it keyed by the `TaskExecutorProcessSpec`?

##########
File path: flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerDriverTest.java
##########
@@ -0,0 +1,544 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.api.common.resources.CPUResource;
+import org.apache.flink.configuration.MemorySize;
+import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.runtime.clusterframework.ApplicationStatus;
+import org.apache.flink.runtime.clusterframework.BootstrapTools;
+import org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.resourcemanager.active.ResourceManagerDriver;
+import org.apache.flink.runtime.resourcemanager.active.ResourceManagerDriverTestBase;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
+import org.apache.flink.runtime.util.HadoopUtils;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.yarn.configuration.YarnResourceManagerConfiguration;
+
+import org.apache.flink.shaded.guava18.com.google.common.collect.ImmutableList;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.flink.configuration.GlobalConfiguration.FLINK_CONF_FILENAME;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_APP_ID;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_CLIENT_HOME_DIR;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_CLIENT_SHIP_FILES;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_FLINK_CLASSPATH;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_HADOOP_USER_NAME;
+import static org.apache.flink.yarn.YarnConfigKeys.FLINK_DIST_JAR;
+import static org.apache.flink.yarn.YarnConfigKeys.FLINK_YARN_FILES;
+import static org.apache.flink.yarn.YarnResourceManagerDriver.ERROR_MESSAGE_ON_SHUTDOWN_REQUEST;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for {@link YarnResourceManagerDriver}.
+ */
+public class YarnResourceManagerDriverTest extends ResourceManagerDriverTestBase<YarnWorkerNode> {
+	private static final Resource testingResource = Resource.newInstance(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+	private static final Container testingContainer = createTestingContainerWithResource(testingResource, 0);
+	private static final TaskExecutorProcessSpec testingTaskExecutorProcessSpec =
+		new TaskExecutorProcessSpec(
+			new CPUResource(1),
+			MemorySize.ZERO,
+			MemorySize.ZERO,
+			MemorySize.ofMebiBytes(256),
+			MemorySize.ofMebiBytes(256),
+			MemorySize.ofMebiBytes(256),
+			MemorySize.ofMebiBytes(256),
+			MemorySize.ZERO,
+			MemorySize.ZERO);
+
+	@Rule
+	public TemporaryFolder folder = new TemporaryFolder();
+
+	@Override
+	protected Context createContext() {
+		return new Context();
+	}
+
+	@Test
+	public void testShutdownRequestCausesFatalError() throws Exception {
+		new Context() {{
+			final CompletableFuture<Throwable> throwableCompletableFuture = new CompletableFuture<>();
+			resourceEventHandlerBuilder.setOnErrorConsumer(throwableCompletableFuture::complete);
+			runTest(() -> {
+				resourceManagerClientCallbackHandler.onShutdownRequest();
+
+				Throwable throwable = throwableCompletableFuture.get(TIMEOUT_SEC, TimeUnit.SECONDS);
+				assertThat(ExceptionUtils.findThrowable(throwable, ResourceManagerException.class).isPresent(), is(true));
+				assertThat(ExceptionUtils.findThrowableWithMessage(throwable, ERROR_MESSAGE_ON_SHUTDOWN_REQUEST).isPresent(), is(true));
+			});
+		}};
+	}
+
+	/**
+	 * Tests that application files are deleted when the YARN application master is de-registered.
+	 */
+	@Test
+	public void testDeleteApplicationFiles() throws Exception {
+		new Context() {{
+			final File applicationDir = folder.newFolder(".flink");
+			env.put(FLINK_YARN_FILES, applicationDir.getCanonicalPath());
+
+			runTest(() -> {
+				getDriver().deregisterApplication(ApplicationStatus.SUCCEEDED, null);
+				assertFalse("YARN application directory was not removed", Files.exists(applicationDir.toPath()));
+			});
+		}};
+	}
+
+	@Test
+	public void testOnContainerCompleted() throws Exception {
+		new Context() {{
+			addContainerRequestFutures.add(new CompletableFuture<>());
+			addContainerRequestFutures.add(new CompletableFuture<>());
+
+			testingYarnAMRMClientAsyncBuilder.setAddContainerRequestConsumer((ignored1, ignored2) ->
+				addContainerRequestFutures.get(addContainerRequestFuturesNumCompleted.getAndIncrement()).complete(null));
+			resourceEventHandlerBuilder.setOnWorkerTerminatedConsumer((ignore1, ignore2) -> getDriver().requestResource(testingTaskExecutorProcessSpec));
+
+			runTest(() -> {
+				runInMainThread(() -> getDriver().requestResource(testingTaskExecutorProcessSpec));
+				resourceManagerClientCallbackHandler.onContainersAllocated(ImmutableList.of(testingContainer));
+
+				verifyFutureCompleted(addContainerRequestFutures.get(0));
+				verifyFutureCompleted(removeContainerRequestFuture);
+				verifyFutureCompleted(startContainerAsyncFuture);
+
+				ContainerStatus testingContainerStatus = createTestingContainerCompletedStatus(testingContainer.getId());
+				resourceManagerClientCallbackHandler.onContainersCompleted(ImmutableList.of(testingContainerStatus));
+
+				verifyFutureCompleted(addContainerRequestFutures.get(1));
+			});
+		}};
+	}
+
+	@Test
+	public void testOnStartContainerError() throws Exception {
+		new Context() {{
+			addContainerRequestFutures.add(new CompletableFuture<>());
+			addContainerRequestFutures.add(new CompletableFuture<>());
+
+			testingYarnAMRMClientAsyncBuilder.setAddContainerRequestConsumer((ignored1, ignored2) ->
+				addContainerRequestFutures.get(addContainerRequestFuturesNumCompleted.getAndIncrement()).complete(null));
+			resourceEventHandlerBuilder.setOnWorkerTerminatedConsumer((ignore1, ignore2) -> getDriver().requestResource(testingTaskExecutorProcessSpec));
+
+			runTest(() -> {
+				runInMainThread(() -> getDriver().requestResource(testingTaskExecutorProcessSpec));
+				resourceManagerClientCallbackHandler.onContainersAllocated(ImmutableList.of(testingContainer));
+
+				verifyFutureCompleted(addContainerRequestFutures.get(0));
+				verifyFutureCompleted(removeContainerRequestFuture);
+				verifyFutureCompleted(startContainerAsyncFuture);

Review comment:
       This part is already covered by `testOnContainerCompleted`

##########
File path: flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerDriverTest.java
##########
@@ -0,0 +1,544 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.api.common.resources.CPUResource;
+import org.apache.flink.configuration.MemorySize;
+import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.runtime.clusterframework.ApplicationStatus;
+import org.apache.flink.runtime.clusterframework.BootstrapTools;
+import org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.resourcemanager.active.ResourceManagerDriver;
+import org.apache.flink.runtime.resourcemanager.active.ResourceManagerDriverTestBase;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
+import org.apache.flink.runtime.util.HadoopUtils;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.yarn.configuration.YarnResourceManagerConfiguration;
+
+import org.apache.flink.shaded.guava18.com.google.common.collect.ImmutableList;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.flink.configuration.GlobalConfiguration.FLINK_CONF_FILENAME;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_APP_ID;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_CLIENT_HOME_DIR;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_CLIENT_SHIP_FILES;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_FLINK_CLASSPATH;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_HADOOP_USER_NAME;
+import static org.apache.flink.yarn.YarnConfigKeys.FLINK_DIST_JAR;
+import static org.apache.flink.yarn.YarnConfigKeys.FLINK_YARN_FILES;
+import static org.apache.flink.yarn.YarnResourceManagerDriver.ERROR_MESSAGE_ON_SHUTDOWN_REQUEST;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for {@link YarnResourceManagerDriver}.
+ */
+public class YarnResourceManagerDriverTest extends ResourceManagerDriverTestBase<YarnWorkerNode> {
+	private static final Resource testingResource = Resource.newInstance(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+	private static final Container testingContainer = createTestingContainerWithResource(testingResource, 0);
+	private static final TaskExecutorProcessSpec testingTaskExecutorProcessSpec =
+		new TaskExecutorProcessSpec(
+			new CPUResource(1),
+			MemorySize.ZERO,
+			MemorySize.ZERO,
+			MemorySize.ofMebiBytes(256),
+			MemorySize.ofMebiBytes(256),
+			MemorySize.ofMebiBytes(256),
+			MemorySize.ofMebiBytes(256),
+			MemorySize.ZERO,
+			MemorySize.ZERO);
+
+	@Rule
+	public TemporaryFolder folder = new TemporaryFolder();
+
+	@Override
+	protected Context createContext() {
+		return new Context();
+	}
+
+	@Test
+	public void testShutdownRequestCausesFatalError() throws Exception {
+		new Context() {{
+			final CompletableFuture<Throwable> throwableCompletableFuture = new CompletableFuture<>();
+			resourceEventHandlerBuilder.setOnErrorConsumer(throwableCompletableFuture::complete);
+			runTest(() -> {
+				resourceManagerClientCallbackHandler.onShutdownRequest();
+
+				Throwable throwable = throwableCompletableFuture.get(TIMEOUT_SEC, TimeUnit.SECONDS);
+				assertThat(ExceptionUtils.findThrowable(throwable, ResourceManagerException.class).isPresent(), is(true));
+				assertThat(ExceptionUtils.findThrowableWithMessage(throwable, ERROR_MESSAGE_ON_SHUTDOWN_REQUEST).isPresent(), is(true));
+			});
+		}};
+	}
+
+	/**
+	 * Tests that application files are deleted when the YARN application master is de-registered.
+	 */
+	@Test
+	public void testDeleteApplicationFiles() throws Exception {
+		new Context() {{
+			final File applicationDir = folder.newFolder(".flink");
+			env.put(FLINK_YARN_FILES, applicationDir.getCanonicalPath());
+
+			runTest(() -> {
+				getDriver().deregisterApplication(ApplicationStatus.SUCCEEDED, null);
+				assertFalse("YARN application directory was not removed", Files.exists(applicationDir.toPath()));
+			});
+		}};
+	}
+
+	@Test
+	public void testOnContainerCompleted() throws Exception {
+		new Context() {{
+			addContainerRequestFutures.add(new CompletableFuture<>());
+			addContainerRequestFutures.add(new CompletableFuture<>());
+
+			testingYarnAMRMClientAsyncBuilder.setAddContainerRequestConsumer((ignored1, ignored2) ->
+				addContainerRequestFutures.get(addContainerRequestFuturesNumCompleted.getAndIncrement()).complete(null));
+			resourceEventHandlerBuilder.setOnWorkerTerminatedConsumer((ignore1, ignore2) -> getDriver().requestResource(testingTaskExecutorProcessSpec));
+
+			runTest(() -> {
+				runInMainThread(() -> getDriver().requestResource(testingTaskExecutorProcessSpec));
+				resourceManagerClientCallbackHandler.onContainersAllocated(ImmutableList.of(testingContainer));
+
+				verifyFutureCompleted(addContainerRequestFutures.get(0));
+				verifyFutureCompleted(removeContainerRequestFuture);
+				verifyFutureCompleted(startContainerAsyncFuture);
+
+				ContainerStatus testingContainerStatus = createTestingContainerCompletedStatus(testingContainer.getId());
+				resourceManagerClientCallbackHandler.onContainersCompleted(ImmutableList.of(testingContainerStatus));
+
+				verifyFutureCompleted(addContainerRequestFutures.get(1));
+			});
+		}};
+	}
+
+	@Test
+	public void testOnStartContainerError() throws Exception {
+		new Context() {{
+			addContainerRequestFutures.add(new CompletableFuture<>());
+			addContainerRequestFutures.add(new CompletableFuture<>());
+
+			testingYarnAMRMClientAsyncBuilder.setAddContainerRequestConsumer((ignored1, ignored2) ->
+				addContainerRequestFutures.get(addContainerRequestFuturesNumCompleted.getAndIncrement()).complete(null));
+			resourceEventHandlerBuilder.setOnWorkerTerminatedConsumer((ignore1, ignore2) -> getDriver().requestResource(testingTaskExecutorProcessSpec));
+
+			runTest(() -> {
+				runInMainThread(() -> getDriver().requestResource(testingTaskExecutorProcessSpec));
+				resourceManagerClientCallbackHandler.onContainersAllocated(ImmutableList.of(testingContainer));
+
+				verifyFutureCompleted(addContainerRequestFutures.get(0));
+				verifyFutureCompleted(removeContainerRequestFuture);
+				verifyFutureCompleted(startContainerAsyncFuture);
+
+				nodeManagerClientCallbackHandler.onStartContainerError(testingContainer.getId(), new Exception("start error"));
+				verifyFutureCompleted(releaseAssignedContainerFuture);
+				verifyFutureCompleted(addContainerRequestFutures.get(1));
+			});
+		}};
+	}
+
+	@Test
+	public void testStartTaskExecutorProcessVariousSpec_SameContainerResource() throws Exception{
+		final TaskExecutorProcessSpec taskExecutorProcessSpec1 =
+			new TaskExecutorProcessSpec(
+				new CPUResource(1),
+				MemorySize.ZERO,
+				MemorySize.ZERO,
+				MemorySize.ofMebiBytes(100),
+				MemorySize.ZERO,
+				MemorySize.ofMebiBytes(100),
+				MemorySize.ofMebiBytes(100),
+				MemorySize.ZERO,
+				MemorySize.ZERO
+			);
+		final TaskExecutorProcessSpec taskExecutorProcessSpec2 =
+			new TaskExecutorProcessSpec(
+				new CPUResource(1),
+				MemorySize.ZERO,
+				MemorySize.ZERO,
+				MemorySize.ofMebiBytes(99),
+				MemorySize.ZERO,
+				MemorySize.ofMebiBytes(100),
+				MemorySize.ofMebiBytes(100),
+				MemorySize.ZERO,
+				MemorySize.ZERO
+			);
+
+		new Context() {{
+			addContainerRequestFutures.add(new CompletableFuture<>());
+			addContainerRequestFutures.add(new CompletableFuture<>());
+			addContainerRequestFutures.add(new CompletableFuture<>());
+			addContainerRequestFutures.add(new CompletableFuture<>());
+
+			final String startCommand1 = TaskManagerOptions.TASK_HEAP_MEMORY.key() + "=" + (100L << 20);
+			final String startCommand2 = TaskManagerOptions.TASK_HEAP_MEMORY.key() + "=" + (99L << 20);
+			final CompletableFuture<Void> startContainerAsyncCommandFuture1 = new CompletableFuture<>();
+			final CompletableFuture<Void> startContainerAsyncCommandFuture2 = new CompletableFuture<>();
+
+			testingYarnAMRMClientAsyncBuilder.setGetMatchingRequestsFunction(ignored ->
+				Collections.singletonList(ImmutableList.of(
+					YarnResourceManagerDriver.getContainerRequest(((YarnResourceManagerDriver) getDriver()).getContainerResource(taskExecutorProcessSpec1).get()),
+					YarnResourceManagerDriver.getContainerRequest(((YarnResourceManagerDriver) getDriver()).getContainerResource(taskExecutorProcessSpec2).get()))));
+			testingYarnAMRMClientAsyncBuilder.setAddContainerRequestConsumer((ignored1, ignored2) ->
+				addContainerRequestFutures.get(addContainerRequestFuturesNumCompleted.getAndIncrement()).complete(null));
+			testingYarnNMClientAsyncBuilder.setStartContainerAsyncConsumer((ignored1, context, ignored2) -> {
+				if (containsStartCommand(context, startCommand1)) {
+					startContainerAsyncCommandFuture1.complete(null);
+				} else if (containsStartCommand(context, startCommand2)) {
+					startContainerAsyncCommandFuture2.complete(null);
+				}
+			});
+			resourceEventHandlerBuilder.setOnWorkerTerminatedConsumer((ignore1, ignore2) -> getDriver().requestResource(taskExecutorProcessSpec1));
+
+			runTest(() -> {
+				final Resource containerResource = ((YarnResourceManagerDriver) getDriver()).getContainerResource(taskExecutorProcessSpec1).get();
+				// Make sure two worker resource spec will be normalized to the same container resource
+				assertEquals(containerResource, ((YarnResourceManagerDriver) getDriver()).getContainerResource(taskExecutorProcessSpec2).get());
+
+				runInMainThread(() -> getDriver().requestResource(taskExecutorProcessSpec1));
+				runInMainThread(() -> getDriver().requestResource(taskExecutorProcessSpec2));
+
+				// Verify both containers requested
+				verifyFutureCompleted(addContainerRequestFutures.get(0));
+				verifyFutureCompleted(addContainerRequestFutures.get(1));
+
+				// Mock that both containers are allocated
+				Container container1 = createTestingContainerWithResource(containerResource);
+				Container container2 = createTestingContainerWithResource(containerResource);
+				resourceManagerClientCallbackHandler.onContainersAllocated(ImmutableList.of(container1, container2));
+
+				// Verify workers with both spec are started.
+				verifyFutureCompleted(startContainerAsyncCommandFuture1);
+				verifyFutureCompleted(startContainerAsyncCommandFuture2);
+
+				// Mock that one container is completed, while the worker is still pending
+				ContainerStatus testingContainerStatus = createTestingContainerCompletedStatus(container1.getId());
+				resourceManagerClientCallbackHandler.onContainersCompleted(Collections.singletonList(testingContainerStatus));
+
+				// Verify that only one more container is requested.
+				verifyFutureCompleted(addContainerRequestFutures.get(2));
+				assertFalse(addContainerRequestFutures.get(3).isDone());
+			});
+		}};
+	}
+
+	@Test
+	public void testStartWorkerVariousSpec_DifferentContainerResource() throws Exception{
+		final TaskExecutorProcessSpec taskExecutorProcessSpec1 =
+			new TaskExecutorProcessSpec(
+				new CPUResource(1),
+				MemorySize.ZERO,
+				MemorySize.ZERO,
+				MemorySize.ofMebiBytes(50),
+				MemorySize.ofMebiBytes(50),
+				MemorySize.ofMebiBytes(50),
+				MemorySize.ofMebiBytes(50),
+				MemorySize.ZERO,
+				MemorySize.ZERO
+			);
+		final TaskExecutorProcessSpec taskExecutorProcessSpec2 =
+			new TaskExecutorProcessSpec(
+				new CPUResource(2),
+				MemorySize.ZERO,
+				MemorySize.ZERO,
+				MemorySize.ofMebiBytes(500),
+				MemorySize.ofMebiBytes(500),
+				MemorySize.ofMebiBytes(500),
+				MemorySize.ofMebiBytes(500),
+				MemorySize.ZERO,
+				MemorySize.ZERO
+			);
+
+		new Context() {{
+			addContainerRequestFutures.add(new CompletableFuture<>());
+			addContainerRequestFutures.add(new CompletableFuture<>());
+			addContainerRequestFutures.add(new CompletableFuture<>());
+			addContainerRequestFutures.add(new CompletableFuture<>());
+
+			final String startCommand1 = TaskManagerOptions.TASK_HEAP_MEMORY.key() + "=" + (50L << 20);
+			final String startCommand2 = TaskManagerOptions.TASK_HEAP_MEMORY.key() + "=" + (100L << 20);
+			final CompletableFuture<Void> startContainerAsyncCommandFuture1 = new CompletableFuture<>();
+			final CompletableFuture<Void> startContainerAsyncCommandFuture2 = new CompletableFuture<>();
+
+			testingYarnAMRMClientAsyncBuilder.setGetMatchingRequestsFunction(tuple -> {
+				if (tuple.f2.getVirtualCores() == 1) {
+					return Collections.singletonList(
+						Collections.singletonList(YarnResourceManagerDriver.getContainerRequest(((YarnResourceManagerDriver) getDriver()).getContainerResource(taskExecutorProcessSpec1).get())));
+				} else if (tuple.f2.getVirtualCores() == 2) {
+					return Collections.singletonList(
+						Collections.singletonList(YarnResourceManagerDriver.getContainerRequest(((YarnResourceManagerDriver) getDriver()).getContainerResource(taskExecutorProcessSpec2).get())));
+				}
+				return null;
+			});
+			testingYarnAMRMClientAsyncBuilder.setAddContainerRequestConsumer((request, ignored) ->
+				addContainerRequestFutures.get(addContainerRequestFuturesNumCompleted.getAndIncrement()).complete(request.getCapability()));
+			testingYarnNMClientAsyncBuilder.setStartContainerAsyncConsumer((ignored1, context, ignored3) -> {
+				if (containsStartCommand(context, startCommand1)) {
+					startContainerAsyncCommandFuture1.complete(null);
+				} else if (containsStartCommand(context, startCommand2)) {
+					startContainerAsyncCommandFuture2.complete(null);
+				}
+			});

Review comment:
       This setup logic looks a bit similar to the setup logic in the previous test. Maybe they can be unified if it is not a lot of work.

##########
File path: flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java
##########
@@ -385,14 +385,14 @@ public void testStopWorkerAfterRegistration() throws Exception {
 			final CompletableFuture<Void> startContainerAsyncFuture = new CompletableFuture<>();
 			final CompletableFuture<Void> stopContainerAsyncFuture = new CompletableFuture<>();
 
-			testingYarnAMRMClientAsync.setGetMatchingRequestsFunction(ignored ->
+			testingYarnAMRMClientAsyncBuilder.setGetMatchingRequestsFunction(ignored ->

Review comment:
       Are we sure that this value will always be seen by the `ResourceManager` which uses the builder when it starts? I believe that this is not given since we are modifying the builder from the testing main thread whereas we later read it from the `RpcService` thread w/o synchronization of any kind.

##########
File path: flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManagerClientFactory.java
##########
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+
+/**
+ * Factory interface for {@link AMRMClientAsync}.
+ */
+public interface YarnResourceManagerClientFactory {
+	AMRMClientAsync<AMRMClient.ContainerRequest> createResourceManagerClient(

Review comment:
       nit: JavaDocs are missing.

##########
File path: flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManagerDriver.java
##########
@@ -0,0 +1,609 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.configuration.TaskManagerOptionsInternal;
+import org.apache.flink.runtime.clusterframework.ApplicationStatus;
+import org.apache.flink.runtime.clusterframework.BootstrapTools;
+import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters;
+import org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.FutureUtils;
+import org.apache.flink.runtime.resourcemanager.active.AbstractResourceManagerDriver;
+import org.apache.flink.runtime.resourcemanager.active.ResourceManagerDriver;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
+import org.apache.flink.runtime.webmonitor.history.HistoryServerUtils;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.yarn.configuration.YarnConfigOptions;
+import org.apache.flink.yarn.configuration.YarnConfigOptionsInternal;
+import org.apache.flink.yarn.configuration.YarnResourceManagerConfiguration;
+
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+/**
+ * Implementation of {@link ResourceManagerDriver} for Yarn deployment.
+ */
+public class YarnResourceManagerDriver extends AbstractResourceManagerDriver<YarnWorkerNode> {
+
+	private static final Priority RM_REQUEST_PRIORITY = Priority.newInstance(1);
+
+	/** Environment variable name of the hostname given by the YARN.
+	 * In task executor we use the hostnames given by YARN consistently throughout akka */
+	private static final String ENV_FLINK_NODE_ID = "_FLINK_NODE_ID";
+
+	static final String ERROR_MESSAGE_ON_SHUTDOWN_REQUEST = "Received shutdown request from YARN ResourceManager.";
+
+	private final YarnConfiguration yarnConfig;
+
+	/** The process environment variables. */
+	private final YarnResourceManagerConfiguration configuration;
+
+	/** Default heartbeat interval between this resource manager and the YARN ResourceManager. */
+	private final int yarnHeartbeatIntervalMillis;
+
+	/** Client to communicate with the Resource Manager (YARN's master). */
+	private AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient;
+
+	/** The heartbeat interval while the resource master is waiting for containers. */
+	private final int containerRequestHeartbeatIntervalMillis;
+
+	/** Client to communicate with the Node manager and launch TaskExecutor processes. */
+	private NMClientAsync nodeManagerClient;

Review comment:
       I would try to not mix final with non-final fields.

##########
File path: flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManagerDriver.java
##########
@@ -0,0 +1,609 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.configuration.TaskManagerOptionsInternal;
+import org.apache.flink.runtime.clusterframework.ApplicationStatus;
+import org.apache.flink.runtime.clusterframework.BootstrapTools;
+import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters;
+import org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.FutureUtils;
+import org.apache.flink.runtime.resourcemanager.active.AbstractResourceManagerDriver;
+import org.apache.flink.runtime.resourcemanager.active.ResourceManagerDriver;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
+import org.apache.flink.runtime.webmonitor.history.HistoryServerUtils;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.yarn.configuration.YarnConfigOptions;
+import org.apache.flink.yarn.configuration.YarnConfigOptionsInternal;
+import org.apache.flink.yarn.configuration.YarnResourceManagerConfiguration;
+
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+/**
+ * Implementation of {@link ResourceManagerDriver} for Yarn deployment.
+ */
+public class YarnResourceManagerDriver extends AbstractResourceManagerDriver<YarnWorkerNode> {
+
+	private static final Priority RM_REQUEST_PRIORITY = Priority.newInstance(1);
+
+	/** Environment variable name of the hostname given by the YARN.
+	 * In task executor we use the hostnames given by YARN consistently throughout akka */
+	private static final String ENV_FLINK_NODE_ID = "_FLINK_NODE_ID";
+
+	static final String ERROR_MESSAGE_ON_SHUTDOWN_REQUEST = "Received shutdown request from YARN ResourceManager.";
+
+	private final YarnConfiguration yarnConfig;
+
+	/** The process environment variables. */
+	private final YarnResourceManagerConfiguration configuration;
+
+	/** Default heartbeat interval between this resource manager and the YARN ResourceManager. */
+	private final int yarnHeartbeatIntervalMillis;
+
+	/** Client to communicate with the Resource Manager (YARN's master). */
+	private AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient;
+
+	/** The heartbeat interval while the resource master is waiting for containers. */
+	private final int containerRequestHeartbeatIntervalMillis;
+
+	/** Client to communicate with the Node manager and launch TaskExecutor processes. */
+	private NMClientAsync nodeManagerClient;
+
+	/** Request resource futures, keyed by container ids. */
+	private final Map<TaskExecutorProcessSpec, Queue<CompletableFuture<YarnWorkerNode>>> requestResourceFutures;
+
+	private final TaskExecutorProcessSpecContainerResourceAdapter taskExecutorProcessSpecContainerResourceAdapter;
+
+	private final RegisterApplicationMasterResponseReflector registerApplicationMasterResponseReflector;
+
+	private TaskExecutorProcessSpecContainerResourceAdapter.MatchingStrategy matchingStrategy;
+
+	private final YarnResourceManagerClientFactory yarnResourceManagerClientFactory;
+
+	private final YarnNodeManagerClientFactory yarnNodeManagerClientFactory;
+
+	public YarnResourceManagerDriver(
+		Configuration flinkConfig,
+		YarnResourceManagerConfiguration configuration,
+		YarnResourceManagerClientFactory yarnResourceManagerClientFactory,
+		YarnNodeManagerClientFactory yarnNodeManagerClientFactory) {
+		super(flinkConfig, GlobalConfiguration.loadConfiguration(configuration.getCurrentDir()));
+
+		this.yarnConfig = new YarnConfiguration();
+		this.requestResourceFutures = new HashMap<>();
+		this.configuration = configuration;
+
+		final int yarnHeartbeatIntervalMS = flinkConfig.getInteger(
+			YarnConfigOptions.HEARTBEAT_DELAY_SECONDS) * 1000;
+
+		final long yarnExpiryIntervalMS = yarnConfig.getLong(
+			YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS,
+			YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS);
+
+		if (yarnHeartbeatIntervalMS >= yarnExpiryIntervalMS) {
+			log.warn("The heartbeat interval of the Flink Application master ({}) is greater " +
+					"than YARN's expiry interval ({}). The application is likely to be killed by YARN.",
+				yarnHeartbeatIntervalMS, yarnExpiryIntervalMS);
+		}
+		yarnHeartbeatIntervalMillis = yarnHeartbeatIntervalMS;
+		containerRequestHeartbeatIntervalMillis = flinkConfig.getInteger(YarnConfigOptions.CONTAINER_REQUEST_HEARTBEAT_INTERVAL_MILLISECONDS);
+
+		this.taskExecutorProcessSpecContainerResourceAdapter = Utils.createTaskExecutorProcessSpecContainerResourceAdapter(flinkConfig, yarnConfig);
+		this.registerApplicationMasterResponseReflector = new RegisterApplicationMasterResponseReflector(log);
+
+		this.matchingStrategy = flinkConfig.getBoolean(YarnConfigOptionsInternal.MATCH_CONTAINER_VCORES) ?
+			TaskExecutorProcessSpecContainerResourceAdapter.MatchingStrategy.MATCH_VCORE :
+			TaskExecutorProcessSpecContainerResourceAdapter.MatchingStrategy.IGNORE_VCORE;
+
+		this.yarnResourceManagerClientFactory = yarnResourceManagerClientFactory;
+		this.yarnNodeManagerClientFactory = yarnNodeManagerClientFactory;
+	}
+
+	// ------------------------------------------------------------------------
+	//  ResourceManagerDriver
+	// ------------------------------------------------------------------------
+
+	@Override
+	protected void initializeInternal() throws Exception {
+		final YarnContainerEventHandler yarnContainerEventHandler = new YarnContainerEventHandler();
+		try {
+			resourceManagerClient = yarnResourceManagerClientFactory.createResourceManagerClient(
+				yarnHeartbeatIntervalMillis,
+				yarnContainerEventHandler);
+			resourceManagerClient.init(yarnConfig);
+			resourceManagerClient.start();
+
+			final RegisterApplicationMasterResponse registerApplicationMasterResponse = registerApplicationMaster();
+			getContainersFromPreviousAttempts(registerApplicationMasterResponse);
+			updateMatchingStrategy(registerApplicationMasterResponse);
+		} catch (Exception e) {
+			throw new ResourceManagerException("Could not start resource manager client.", e);
+		}
+
+		nodeManagerClient = yarnNodeManagerClientFactory.createNodeManagerClient(yarnContainerEventHandler);
+		nodeManagerClient.init(yarnConfig);
+		nodeManagerClient.start();
+	}
+
+	@Override
+	public CompletableFuture<Void> terminate() {
+		// shut down all components
+		Exception exception = null;
+
+		if (resourceManagerClient != null) {
+			try {
+				resourceManagerClient.stop();
+			} catch (Exception e) {
+				exception = e;
+			}
+		}
+
+		if (nodeManagerClient != null) {
+			try {
+				nodeManagerClient.stop();
+			} catch (Exception e) {
+				exception = ExceptionUtils.firstOrSuppressed(e, exception);
+			}
+		}
+
+		return exception == null ?
+			FutureUtils.completedVoidFuture() :
+			FutureUtils.completedExceptionally(exception);
+	}
+
+	@Override
+	public void deregisterApplication(ApplicationStatus finalStatus, @Nullable String optionalDiagnostics) {
+		// first, de-register from YARN
+		final FinalApplicationStatus yarnStatus = getYarnStatus(finalStatus);
+		log.info("Unregister application from the YARN Resource Manager with final status {}.", yarnStatus);
+
+		final Optional<URL> historyServerURL = HistoryServerUtils.getHistoryServerURL(flinkConfig);
+
+		final String appTrackingUrl = historyServerURL.map(URL::toString).orElse("");
+
+		try {
+			resourceManagerClient.unregisterApplicationMaster(yarnStatus, optionalDiagnostics, appTrackingUrl);
+		} catch (Throwable t) {

Review comment:
       We should probably only catch the thrown exceptions and not all `Throwables` here.

##########
File path: flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManagerDriver.java
##########
@@ -72,354 +62,237 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Queue;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 
 /**
- * The yarn implementation of the resource manager. Used when the system is started
- * via the resource framework YARN.
+ * Implementation of {@link ResourceManagerDriver} for Kubernetes deployment.
  */
-public class YarnResourceManager extends LegacyActiveResourceManager<YarnWorkerNode>
-		implements AMRMClientAsync.CallbackHandler, NMClientAsync.CallbackHandler {
+public class YarnResourceManagerDriver extends AbstractResourceManagerDriver<YarnWorkerNode> {
 
 	private static final Priority RM_REQUEST_PRIORITY = Priority.newInstance(1);
 
-	/** YARN container map. */
-	private final ConcurrentMap<ResourceID, YarnWorkerNode> workerNodeMap;
-
 	/** Environment variable name of the hostname given by the YARN.
 	 * In task executor we use the hostnames given by YARN consistently throughout akka */
 	static final String ENV_FLINK_NODE_ID = "_FLINK_NODE_ID";
 
 	static final String ERROR_MASSAGE_ON_SHUTDOWN_REQUEST = "Received shutdown request from YARN ResourceManager.";
 
-	/** Default heartbeat interval between this resource manager and the YARN ResourceManager. */
-	private final int yarnHeartbeatIntervalMillis;
-
 	private final YarnConfiguration yarnConfig;
 
-	@Nullable
-	private final String webInterfaceUrl;
+	/** The process environment variables. */
+	private final YarnResourceManagerDriverConfiguration configuration;
 
-	/** The heartbeat interval while the resource master is waiting for containers. */
-	private final int containerRequestHeartbeatIntervalMillis;
+	/** Default heartbeat interval between this resource manager and the YARN ResourceManager. */
+	private final int yarnHeartbeatIntervalMillis;
 
 	/** Client to communicate with the Resource Manager (YARN's master). */
 	private AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient;
 
+	/** The heartbeat interval while the resource master is waiting for containers. */
+	private final int containerRequestHeartbeatIntervalMillis;
+
 	/** Client to communicate with the Node manager and launch TaskExecutor processes. */
 	private NMClientAsync nodeManagerClient;
 
-	private final WorkerSpecContainerResourceAdapter workerSpecContainerResourceAdapter;
+	/** Request resource futures, keyed by container ids. */
+	private final Map<TaskExecutorProcessSpec, Queue<CompletableFuture<YarnWorkerNode>>> requestResourceFutures;
+
+	private final TaskExecutorProcessSpecContainerResourceAdapter taskExecutorProcessSpecContainerResourceAdapter;
 
 	private final RegisterApplicationMasterResponseReflector registerApplicationMasterResponseReflector;
 
-	private WorkerSpecContainerResourceAdapter.MatchingStrategy matchingStrategy;
-
-	public YarnResourceManager(
-			RpcService rpcService,
-			ResourceID resourceId,
-			Configuration flinkConfig,
-			Map<String, String> env,
-			HighAvailabilityServices highAvailabilityServices,
-			HeartbeatServices heartbeatServices,
-			SlotManager slotManager,
-			ResourceManagerPartitionTrackerFactory clusterPartitionTrackerFactory,
-			JobLeaderIdService jobLeaderIdService,
-			ClusterInformation clusterInformation,
-			FatalErrorHandler fatalErrorHandler,
-			@Nullable String webInterfaceUrl,
-			ResourceManagerMetricGroup resourceManagerMetricGroup) {
-		super(
-			flinkConfig,
-			env,
-			rpcService,
-			resourceId,
-			highAvailabilityServices,
-			heartbeatServices,
-			slotManager,
-			clusterPartitionTrackerFactory,
-			jobLeaderIdService,
-			clusterInformation,
-			fatalErrorHandler,
-			resourceManagerMetricGroup);
+	private TaskExecutorProcessSpecContainerResourceAdapter.MatchingStrategy matchingStrategy;
+
+	private final YarnResourceManagerClientFactory yarnResourceManagerClientFactory;
+
+	private final YarnNodeManagerClientFactory yarnNodeManagerClientFactory;
+
+	public YarnResourceManagerDriver(
+		Configuration flinkConfig,
+		YarnResourceManagerDriverConfiguration configuration,
+		YarnResourceManagerClientFactory yarnResourceManagerClientFactory,
+		YarnNodeManagerClientFactory yarnNodeManagerClientFactory) {
+		super(flinkConfig, GlobalConfiguration.loadConfiguration(configuration.getCurrentDir()));
+
 		this.yarnConfig = new YarnConfiguration();
-		this.workerNodeMap = new ConcurrentHashMap<>();
+		this.requestResourceFutures = new HashMap<>();
+		this.configuration = configuration;
+
 		final int yarnHeartbeatIntervalMS = flinkConfig.getInteger(
-				YarnConfigOptions.HEARTBEAT_DELAY_SECONDS) * 1000;
+			YarnConfigOptions.HEARTBEAT_DELAY_SECONDS) * 1000;
 
 		final long yarnExpiryIntervalMS = yarnConfig.getLong(
-				YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS,
-				YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS);
+			YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS,
+			YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS);
 
 		if (yarnHeartbeatIntervalMS >= yarnExpiryIntervalMS) {
 			log.warn("The heartbeat interval of the Flink Application master ({}) is greater " +
 					"than YARN's expiry interval ({}). The application is likely to be killed by YARN.",
-					yarnHeartbeatIntervalMS, yarnExpiryIntervalMS);
+				yarnHeartbeatIntervalMS, yarnExpiryIntervalMS);
 		}
 		yarnHeartbeatIntervalMillis = yarnHeartbeatIntervalMS;
 		containerRequestHeartbeatIntervalMillis = flinkConfig.getInteger(YarnConfigOptions.CONTAINER_REQUEST_HEARTBEAT_INTERVAL_MILLISECONDS);
 
-		this.webInterfaceUrl = webInterfaceUrl;
-
-		this.workerSpecContainerResourceAdapter = Utils.createWorkerSpecContainerResourceAdapter(flinkConfig, yarnConfig);
+		this.taskExecutorProcessSpecContainerResourceAdapter = Utils.createTaskExecutorProcessSpecContainerResourceAdapter(flinkConfig, yarnConfig);
 		this.registerApplicationMasterResponseReflector = new RegisterApplicationMasterResponseReflector(log);
 
 		this.matchingStrategy = flinkConfig.getBoolean(YarnConfigOptionsInternal.MATCH_CONTAINER_VCORES) ?
-			WorkerSpecContainerResourceAdapter.MatchingStrategy.MATCH_VCORE :
-			WorkerSpecContainerResourceAdapter.MatchingStrategy.IGNORE_VCORE;
-	}
-
-	protected AMRMClientAsync<AMRMClient.ContainerRequest> createAndStartResourceManagerClient(
-			YarnConfiguration yarnConfiguration,
-			int yarnHeartbeatIntervalMillis,
-			@Nullable String webInterfaceUrl) throws Exception {
-		AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = AMRMClientAsync.createAMRMClientAsync(
-			yarnHeartbeatIntervalMillis,
-			this);
-
-		resourceManagerClient.init(yarnConfiguration);
-		resourceManagerClient.start();
-
-		//TODO: change akka address to tcp host and port, the getAddress() interface should return a standard tcp address
-		Tuple2<String, Integer> hostPort = parseHostPort(getAddress());
-
-		final int restPort;
-
-		if (webInterfaceUrl != null) {
-			final int lastColon = webInterfaceUrl.lastIndexOf(':');
-
-			if (lastColon == -1) {
-				restPort = -1;
-			} else {
-				restPort = Integer.valueOf(webInterfaceUrl.substring(lastColon + 1));
-			}
-		} else {
-			restPort = -1;
-		}
-
-		final RegisterApplicationMasterResponse registerApplicationMasterResponse =
-			resourceManagerClient.registerApplicationMaster(hostPort.f0, restPort, webInterfaceUrl);
-		getContainersFromPreviousAttempts(registerApplicationMasterResponse);
-		updateMatchingStrategy(registerApplicationMasterResponse);
+			TaskExecutorProcessSpecContainerResourceAdapter.MatchingStrategy.MATCH_VCORE :
+			TaskExecutorProcessSpecContainerResourceAdapter.MatchingStrategy.IGNORE_VCORE;
 
-		return resourceManagerClient;
+		this.yarnResourceManagerClientFactory = yarnResourceManagerClientFactory;
+		this.yarnNodeManagerClientFactory = yarnNodeManagerClientFactory;
 	}
 
-	private void getContainersFromPreviousAttempts(final RegisterApplicationMasterResponse registerApplicationMasterResponse) {
-		final List<Container> containersFromPreviousAttempts =
-			registerApplicationMasterResponseReflector.getContainersFromPreviousAttempts(registerApplicationMasterResponse);
-
-		log.info("Recovered {} containers from previous attempts ({}).", containersFromPreviousAttempts.size(), containersFromPreviousAttempts);
-
-		for (final Container container : containersFromPreviousAttempts) {
-			final ResourceID resourceID = getContainerResourceId(container);
-			workerNodeMap.put(resourceID, new YarnWorkerNode(container, resourceID));
-		}
-	}
-
-	private void updateMatchingStrategy(final RegisterApplicationMasterResponse registerApplicationMasterResponse) {
-		final Optional<Set<String>> schedulerResourceTypesOptional =
-			registerApplicationMasterResponseReflector.getSchedulerResourceTypeNames(registerApplicationMasterResponse);
-
-		final WorkerSpecContainerResourceAdapter.MatchingStrategy strategy;
-		if (schedulerResourceTypesOptional.isPresent()) {
-			Set<String> types = schedulerResourceTypesOptional.get();
-			log.info("Register application master response contains scheduler resource types: {}.", types);
-			matchingStrategy = types.contains("CPU") ?
-				WorkerSpecContainerResourceAdapter.MatchingStrategy.MATCH_VCORE :
-				WorkerSpecContainerResourceAdapter.MatchingStrategy.IGNORE_VCORE;
-		} else {
-			log.info("Register application master response does not contain scheduler resource types, use '{}'.",
-				YarnConfigOptionsInternal.MATCH_CONTAINER_VCORES.key());
-		}
-		log.info("Container matching strategy: {}.", matchingStrategy);
-	}
-
-	protected NMClientAsync createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
-		// create the client to communicate with the node managers
-		NMClientAsync nodeManagerClient = NMClientAsync.createNMClientAsync(this);
-		nodeManagerClient.init(yarnConfiguration);
-		nodeManagerClient.start();
-		return nodeManagerClient;
-	}
-
-	@Override
-	protected Configuration loadClientConfiguration() {
-		return GlobalConfiguration.loadConfiguration(env.get(ApplicationConstants.Environment.PWD.key()));
-	}
+	// ------------------------------------------------------------------------
+	//  ResourceManagerDriver
+	// ------------------------------------------------------------------------
 
 	@Override
-	protected void initialize() throws ResourceManagerException {
+	protected void initializeInternal() throws Exception {
+		final YarnContainerEventHandler yarnContainerEventHandler = new YarnContainerEventHandler();
 		try {
-			resourceManagerClient = createAndStartResourceManagerClient(
-				yarnConfig,
+			resourceManagerClient = yarnResourceManagerClientFactory.createResourceManagerClient(
 				yarnHeartbeatIntervalMillis,
-				webInterfaceUrl);
+				yarnContainerEventHandler);
+			resourceManagerClient.init(yarnConfig);
+			resourceManagerClient.start();
+
+			final RegisterApplicationMasterResponse registerApplicationMasterResponse = registerApplicationMaster();
+			getContainersFromPreviousAttempts(registerApplicationMasterResponse);
+			updateMatchingStrategy(registerApplicationMasterResponse);
 		} catch (Exception e) {

Review comment:
       I guess one could either narrow the scope of the try block to `registerApplicationMaster`, get rid of the try block or extending it over the whole `initializeInternal` method, to make it consistent.

##########
File path: flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManagerDriver.java
##########
@@ -0,0 +1,609 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.configuration.TaskManagerOptionsInternal;
+import org.apache.flink.runtime.clusterframework.ApplicationStatus;
+import org.apache.flink.runtime.clusterframework.BootstrapTools;
+import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters;
+import org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.concurrent.FutureUtils;
+import org.apache.flink.runtime.resourcemanager.active.AbstractResourceManagerDriver;
+import org.apache.flink.runtime.resourcemanager.active.ResourceManagerDriver;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
+import org.apache.flink.runtime.webmonitor.history.HistoryServerUtils;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.yarn.configuration.YarnConfigOptions;
+import org.apache.flink.yarn.configuration.YarnConfigOptionsInternal;
+import org.apache.flink.yarn.configuration.YarnResourceManagerConfiguration;
+
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+/**
+ * Implementation of {@link ResourceManagerDriver} for Yarn deployment.
+ */
+public class YarnResourceManagerDriver extends AbstractResourceManagerDriver<YarnWorkerNode> {
+
+	private static final Priority RM_REQUEST_PRIORITY = Priority.newInstance(1);
+
+	/** Environment variable name of the hostname given by the YARN.
+	 * In task executor we use the hostnames given by YARN consistently throughout akka */
+	private static final String ENV_FLINK_NODE_ID = "_FLINK_NODE_ID";
+
+	static final String ERROR_MESSAGE_ON_SHUTDOWN_REQUEST = "Received shutdown request from YARN ResourceManager.";
+
+	private final YarnConfiguration yarnConfig;
+
+	/** The process environment variables. */
+	private final YarnResourceManagerConfiguration configuration;
+
+	/** Default heartbeat interval between this resource manager and the YARN ResourceManager. */
+	private final int yarnHeartbeatIntervalMillis;
+
+	/** Client to communicate with the Resource Manager (YARN's master). */
+	private AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient;
+
+	/** The heartbeat interval while the resource master is waiting for containers. */
+	private final int containerRequestHeartbeatIntervalMillis;
+
+	/** Client to communicate with the Node manager and launch TaskExecutor processes. */
+	private NMClientAsync nodeManagerClient;
+
+	/** Request resource futures, keyed by container ids. */
+	private final Map<TaskExecutorProcessSpec, Queue<CompletableFuture<YarnWorkerNode>>> requestResourceFutures;
+
+	private final TaskExecutorProcessSpecContainerResourceAdapter taskExecutorProcessSpecContainerResourceAdapter;
+
+	private final RegisterApplicationMasterResponseReflector registerApplicationMasterResponseReflector;
+
+	private TaskExecutorProcessSpecContainerResourceAdapter.MatchingStrategy matchingStrategy;
+
+	private final YarnResourceManagerClientFactory yarnResourceManagerClientFactory;
+
+	private final YarnNodeManagerClientFactory yarnNodeManagerClientFactory;
+
+	public YarnResourceManagerDriver(
+		Configuration flinkConfig,
+		YarnResourceManagerConfiguration configuration,
+		YarnResourceManagerClientFactory yarnResourceManagerClientFactory,
+		YarnNodeManagerClientFactory yarnNodeManagerClientFactory) {
+		super(flinkConfig, GlobalConfiguration.loadConfiguration(configuration.getCurrentDir()));
+
+		this.yarnConfig = new YarnConfiguration();
+		this.requestResourceFutures = new HashMap<>();
+		this.configuration = configuration;
+
+		final int yarnHeartbeatIntervalMS = flinkConfig.getInteger(
+			YarnConfigOptions.HEARTBEAT_DELAY_SECONDS) * 1000;
+
+		final long yarnExpiryIntervalMS = yarnConfig.getLong(
+			YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS,
+			YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS);
+
+		if (yarnHeartbeatIntervalMS >= yarnExpiryIntervalMS) {
+			log.warn("The heartbeat interval of the Flink Application master ({}) is greater " +
+					"than YARN's expiry interval ({}). The application is likely to be killed by YARN.",
+				yarnHeartbeatIntervalMS, yarnExpiryIntervalMS);
+		}
+		yarnHeartbeatIntervalMillis = yarnHeartbeatIntervalMS;
+		containerRequestHeartbeatIntervalMillis = flinkConfig.getInteger(YarnConfigOptions.CONTAINER_REQUEST_HEARTBEAT_INTERVAL_MILLISECONDS);
+
+		this.taskExecutorProcessSpecContainerResourceAdapter = Utils.createTaskExecutorProcessSpecContainerResourceAdapter(flinkConfig, yarnConfig);
+		this.registerApplicationMasterResponseReflector = new RegisterApplicationMasterResponseReflector(log);
+
+		this.matchingStrategy = flinkConfig.getBoolean(YarnConfigOptionsInternal.MATCH_CONTAINER_VCORES) ?
+			TaskExecutorProcessSpecContainerResourceAdapter.MatchingStrategy.MATCH_VCORE :
+			TaskExecutorProcessSpecContainerResourceAdapter.MatchingStrategy.IGNORE_VCORE;
+
+		this.yarnResourceManagerClientFactory = yarnResourceManagerClientFactory;
+		this.yarnNodeManagerClientFactory = yarnNodeManagerClientFactory;
+	}
+
+	// ------------------------------------------------------------------------
+	//  ResourceManagerDriver
+	// ------------------------------------------------------------------------
+
+	@Override
+	protected void initializeInternal() throws Exception {
+		final YarnContainerEventHandler yarnContainerEventHandler = new YarnContainerEventHandler();
+		try {
+			resourceManagerClient = yarnResourceManagerClientFactory.createResourceManagerClient(
+				yarnHeartbeatIntervalMillis,
+				yarnContainerEventHandler);
+			resourceManagerClient.init(yarnConfig);
+			resourceManagerClient.start();
+
+			final RegisterApplicationMasterResponse registerApplicationMasterResponse = registerApplicationMaster();
+			getContainersFromPreviousAttempts(registerApplicationMasterResponse);
+			updateMatchingStrategy(registerApplicationMasterResponse);
+		} catch (Exception e) {
+			throw new ResourceManagerException("Could not start resource manager client.", e);
+		}
+
+		nodeManagerClient = yarnNodeManagerClientFactory.createNodeManagerClient(yarnContainerEventHandler);
+		nodeManagerClient.init(yarnConfig);
+		nodeManagerClient.start();
+	}
+
+	@Override
+	public CompletableFuture<Void> terminate() {
+		// shut down all components
+		Exception exception = null;
+
+		if (resourceManagerClient != null) {
+			try {
+				resourceManagerClient.stop();
+			} catch (Exception e) {
+				exception = e;
+			}
+		}
+
+		if (nodeManagerClient != null) {
+			try {
+				nodeManagerClient.stop();
+			} catch (Exception e) {
+				exception = ExceptionUtils.firstOrSuppressed(e, exception);
+			}
+		}
+
+		return exception == null ?
+			FutureUtils.completedVoidFuture() :
+			FutureUtils.completedExceptionally(exception);
+	}
+
+	@Override
+	public void deregisterApplication(ApplicationStatus finalStatus, @Nullable String optionalDiagnostics) {
+		// first, de-register from YARN
+		final FinalApplicationStatus yarnStatus = getYarnStatus(finalStatus);
+		log.info("Unregister application from the YARN Resource Manager with final status {}.", yarnStatus);
+
+		final Optional<URL> historyServerURL = HistoryServerUtils.getHistoryServerURL(flinkConfig);
+
+		final String appTrackingUrl = historyServerURL.map(URL::toString).orElse("");
+
+		try {
+			resourceManagerClient.unregisterApplicationMaster(yarnStatus, optionalDiagnostics, appTrackingUrl);
+		} catch (Throwable t) {
+			log.error("Could not unregister the application master.", t);
+		}
+
+		Utils.deleteApplicationFiles(configuration.getYarnFiles());
+	}
+
+	@Override
+	public CompletableFuture<YarnWorkerNode> requestResource(TaskExecutorProcessSpec taskExecutorProcessSpec) {
+		final Optional<Resource> containerResourceOptional = getContainerResource(taskExecutorProcessSpec);
+		final CompletableFuture<YarnWorkerNode> requestResourceFuture = new CompletableFuture<>();
+
+		if (containerResourceOptional.isPresent()) {
+			resourceManagerClient.addContainerRequest(getContainerRequest(containerResourceOptional.get()));
+
+			// make sure we transmit the request fast and receive fast news of granted allocations
+			resourceManagerClient.setHeartbeatInterval(containerRequestHeartbeatIntervalMillis);
+
+			requestResourceFutures.computeIfAbsent(taskExecutorProcessSpec, ignore -> new LinkedList<>()).add(requestResourceFuture);
+
+			log.info("Requesting new TaskExecutor container with resource {}.", taskExecutorProcessSpec);
+		} else {
+			requestResourceFuture.completeExceptionally(
+				new ResourceManagerException(String.format("Could not compute the container Resource from the given TaskExecutorProcessSpec %s.", taskExecutorProcessSpec)));
+		}
+
+		return requestResourceFuture;
+	}
+
+	@Override
+	public void releaseResource(YarnWorkerNode workerNode) {
+		final Container container = workerNode.getContainer();
+		log.info("Stopping container {}.", workerNode.getResourceID().getStringWithMetadata());
+		nodeManagerClient.stopContainerAsync(container.getId(), container.getNodeId());
+		resourceManagerClient.releaseAssignedContainer(container.getId());
+	}
+
+	// ------------------------------------------------------------------------
+	//  Internal
+	// ------------------------------------------------------------------------
+
+	private void onContainersOfResourceAllocated(Resource resource, List<Container> containers) {
+		final List<TaskExecutorProcessSpec> pendingTaskExecutorProcessSpecs =
+			taskExecutorProcessSpecContainerResourceAdapter.getTaskExecutorProcessSpec(resource, matchingStrategy).stream()
+				.flatMap(spec -> Collections.nCopies(getNumRequestedNotAllocatedWorkersFor(spec), spec).stream())
+				.collect(Collectors.toList());
+
+		int numPending = pendingTaskExecutorProcessSpecs.size();
+		log.info("Received {} containers with resource {}, {} pending container requests.",
+			containers.size(),
+			resource,
+			numPending);
+
+		final Iterator<Container> containerIterator = containers.iterator();
+		final Iterator<TaskExecutorProcessSpec> pendingTaskExecutorProcessSpecIterator = pendingTaskExecutorProcessSpecs.iterator();
+		final Iterator<AMRMClient.ContainerRequest> pendingRequestsIterator =
+			getPendingRequestsAndCheckConsistency(resource, pendingTaskExecutorProcessSpecs.size()).iterator();
+
+		int numAccepted = 0;
+		while (containerIterator.hasNext() && pendingTaskExecutorProcessSpecIterator.hasNext()) {
+			final TaskExecutorProcessSpec taskExecutorProcessSpec = pendingTaskExecutorProcessSpecIterator.next();
+			final Container container = containerIterator.next();
+			final AMRMClient.ContainerRequest pendingRequest = pendingRequestsIterator.next();
+			final ResourceID resourceId = getContainerResourceId(container);
+			final CompletableFuture<YarnWorkerNode> requestResourceFuture =
+				Preconditions.checkNotNull(
+					Preconditions.checkNotNull(
+						requestResourceFutures.get(taskExecutorProcessSpec),
+						"The requestResourceFuture for TasExecutorProcessSpec %s should not be null.", taskExecutorProcessSpec).poll(),
+					"The requestResourceFuture queue for TasExecutorProcessSpec %s should not be empty.", taskExecutorProcessSpec);
+			if (requestResourceFutures.get(taskExecutorProcessSpec).isEmpty()) {
+				requestResourceFutures.remove(taskExecutorProcessSpec);
+			}
+
+			startTaskExecutorInContainer(container, taskExecutorProcessSpec, resourceId, requestResourceFuture);
+			removeContainerRequest(pendingRequest);
+
+			numAccepted++;
+		}
+		numPending -= numAccepted;
+
+		int numExcess = 0;
+		while (containerIterator.hasNext()) {
+			returnExcessContainer(containerIterator.next());
+			numExcess++;
+		}
+
+		log.info("Accepted {} requested containers, returned {} excess containers, {} pending container requests of resource {}.",
+			numAccepted, numExcess, numPending, resource);
+	}
+
+	private int getNumRequestedNotAllocatedWorkers() {
+		return requestResourceFutures.values().stream().mapToInt(Queue::size).sum();
+	}
+
+	private int getNumRequestedNotAllocatedWorkersFor(TaskExecutorProcessSpec taskExecutorProcessSpec) {
+		return requestResourceFutures.getOrDefault(taskExecutorProcessSpec, new LinkedList<>()).size();
+	}
+
+	private void removeContainerRequest(AMRMClient.ContainerRequest pendingContainerRequest) {
+		log.info("Removing container request {}.", pendingContainerRequest);
+		resourceManagerClient.removeContainerRequest(pendingContainerRequest);
+	}
+
+	private void returnExcessContainer(Container excessContainer) {
+		log.info("Returning excess container {}.", excessContainer.getId());
+		resourceManagerClient.releaseAssignedContainer(excessContainer.getId());
+	}
+
+	private void startTaskExecutorInContainer(Container container, TaskExecutorProcessSpec taskExecutorProcessSpec, ResourceID resourceId, CompletableFuture<YarnWorkerNode> requestResourceFuture) {
+		final YarnWorkerNode yarnWorkerNode = new YarnWorkerNode(container, resourceId);
+
+		try {
+			// Context information used to start a TaskExecutor Java process
+			ContainerLaunchContext taskExecutorLaunchContext = createTaskExecutorLaunchContext(
+				resourceId,
+				container.getNodeId().getHost(),
+				taskExecutorProcessSpec);
+
+			nodeManagerClient.startContainerAsync(container, taskExecutorLaunchContext);
+			requestResourceFuture.complete(yarnWorkerNode);
+		} catch (Throwable t) {
+			requestResourceFuture.completeExceptionally(t);
+		}
+	}
+
+	private Collection<AMRMClient.ContainerRequest> getPendingRequestsAndCheckConsistency(Resource resource, int expectedNum) {
+		final Collection<Resource> equivalentResources = taskExecutorProcessSpecContainerResourceAdapter.getEquivalentContainerResource(resource, matchingStrategy);
+		final List<? extends Collection<AMRMClient.ContainerRequest>> matchingRequests =
+			equivalentResources.stream()
+				.flatMap(equivalentResource -> resourceManagerClient.getMatchingRequests(
+					RM_REQUEST_PRIORITY,
+					ResourceRequest.ANY,
+					equivalentResource).stream())
+				.collect(Collectors.toList());
+
+		final Collection<AMRMClient.ContainerRequest> matchingContainerRequests;
+
+		if (matchingRequests.isEmpty()) {
+			matchingContainerRequests = Collections.emptyList();
+		} else {
+			final Collection<AMRMClient.ContainerRequest> collection = matchingRequests.get(0);
+			matchingContainerRequests = new ArrayList<>(collection);
+		}

Review comment:
       I know that this method has been copied from `YarnResourceManager`. However, I am not sure whether this method is still correct given that we can add container requests with different resources. The original method has been written with a constant `Resource` in mind. Hence, instead of taking the first `matchingRequests.get(0)`, I believe that one needs to add the collections up.

##########
File path: flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerDriverTest.java
##########
@@ -0,0 +1,544 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.api.common.resources.CPUResource;
+import org.apache.flink.configuration.MemorySize;
+import org.apache.flink.configuration.TaskManagerOptions;
+import org.apache.flink.runtime.clusterframework.ApplicationStatus;
+import org.apache.flink.runtime.clusterframework.BootstrapTools;
+import org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.resourcemanager.active.ResourceManagerDriver;
+import org.apache.flink.runtime.resourcemanager.active.ResourceManagerDriverTestBase;
+import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException;
+import org.apache.flink.runtime.util.HadoopUtils;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.yarn.configuration.YarnResourceManagerConfiguration;
+
+import org.apache.flink.shaded.guava18.com.google.common.collect.ImmutableList;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.flink.configuration.GlobalConfiguration.FLINK_CONF_FILENAME;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_APP_ID;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_CLIENT_HOME_DIR;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_CLIENT_SHIP_FILES;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_FLINK_CLASSPATH;
+import static org.apache.flink.yarn.YarnConfigKeys.ENV_HADOOP_USER_NAME;
+import static org.apache.flink.yarn.YarnConfigKeys.FLINK_DIST_JAR;
+import static org.apache.flink.yarn.YarnConfigKeys.FLINK_YARN_FILES;
+import static org.apache.flink.yarn.YarnResourceManagerDriver.ERROR_MESSAGE_ON_SHUTDOWN_REQUEST;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for {@link YarnResourceManagerDriver}.
+ */
+public class YarnResourceManagerDriverTest extends ResourceManagerDriverTestBase<YarnWorkerNode> {
+	private static final Resource testingResource = Resource.newInstance(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+	private static final Container testingContainer = createTestingContainerWithResource(testingResource, 0);
+	private static final TaskExecutorProcessSpec testingTaskExecutorProcessSpec =
+		new TaskExecutorProcessSpec(
+			new CPUResource(1),
+			MemorySize.ZERO,
+			MemorySize.ZERO,
+			MemorySize.ofMebiBytes(256),
+			MemorySize.ofMebiBytes(256),
+			MemorySize.ofMebiBytes(256),
+			MemorySize.ofMebiBytes(256),
+			MemorySize.ZERO,
+			MemorySize.ZERO);
+
+	@Rule
+	public TemporaryFolder folder = new TemporaryFolder();
+
+	@Override
+	protected Context createContext() {
+		return new Context();
+	}
+
+	@Test
+	public void testShutdownRequestCausesFatalError() throws Exception {
+		new Context() {{
+			final CompletableFuture<Throwable> throwableCompletableFuture = new CompletableFuture<>();
+			resourceEventHandlerBuilder.setOnErrorConsumer(throwableCompletableFuture::complete);
+			runTest(() -> {
+				resourceManagerClientCallbackHandler.onShutdownRequest();
+
+				Throwable throwable = throwableCompletableFuture.get(TIMEOUT_SEC, TimeUnit.SECONDS);
+				assertThat(ExceptionUtils.findThrowable(throwable, ResourceManagerException.class).isPresent(), is(true));
+				assertThat(ExceptionUtils.findThrowableWithMessage(throwable, ERROR_MESSAGE_ON_SHUTDOWN_REQUEST).isPresent(), is(true));
+			});
+		}};
+	}
+
+	/**
+	 * Tests that application files are deleted when the YARN application master is de-registered.
+	 */
+	@Test
+	public void testDeleteApplicationFiles() throws Exception {
+		new Context() {{
+			final File applicationDir = folder.newFolder(".flink");
+			env.put(FLINK_YARN_FILES, applicationDir.getCanonicalPath());
+
+			runTest(() -> {
+				getDriver().deregisterApplication(ApplicationStatus.SUCCEEDED, null);
+				assertFalse("YARN application directory was not removed", Files.exists(applicationDir.toPath()));
+			});
+		}};
+	}
+
+	@Test
+	public void testOnContainerCompleted() throws Exception {
+		new Context() {{
+			addContainerRequestFutures.add(new CompletableFuture<>());
+			addContainerRequestFutures.add(new CompletableFuture<>());
+
+			testingYarnAMRMClientAsyncBuilder.setAddContainerRequestConsumer((ignored1, ignored2) ->
+				addContainerRequestFutures.get(addContainerRequestFuturesNumCompleted.getAndIncrement()).complete(null));
+			resourceEventHandlerBuilder.setOnWorkerTerminatedConsumer((ignore1, ignore2) -> getDriver().requestResource(testingTaskExecutorProcessSpec));
+
+			runTest(() -> {
+				runInMainThread(() -> getDriver().requestResource(testingTaskExecutorProcessSpec));
+				resourceManagerClientCallbackHandler.onContainersAllocated(ImmutableList.of(testingContainer));
+
+				verifyFutureCompleted(addContainerRequestFutures.get(0));
+				verifyFutureCompleted(removeContainerRequestFuture);
+				verifyFutureCompleted(startContainerAsyncFuture);
+
+				ContainerStatus testingContainerStatus = createTestingContainerCompletedStatus(testingContainer.getId());
+				resourceManagerClientCallbackHandler.onContainersCompleted(ImmutableList.of(testingContainerStatus));
+
+				verifyFutureCompleted(addContainerRequestFutures.get(1));

Review comment:
       This could actually also be a separate test.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org