You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2018/01/18 23:47:46 UTC
[15/49] hadoop git commit: YARN-5366. Improve signal handling and
delete delay for Docker on Yarn. (Contributed by Shane Kumpf)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d65dbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index 18d1dce..9602142 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerStateTransitionListener;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitor;
@@ -808,5 +809,10 @@ public abstract class BaseAMRMProxyTest {
public NodeManagerMetrics getNodeManagerMetrics() {
return null;
}
+
+ @Override
+ public DeletionService getDeletionService() {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d65dbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/TestNMProtoUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/TestNMProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/TestNMProtoUtils.java
index 69e01bc..11a69e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/TestNMProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/TestNMProtoUtils.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.recovery.DeletionTaskRecoveryInfo;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DeletionTask;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DeletionTaskType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DockerContainerDeletionTask;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionTask;
import org.junit.Test;
@@ -77,6 +78,29 @@ public class TestNMProtoUtils {
}
@Test
+ public void testConvertProtoToDockerContainerDeletionTask() throws Exception {
+ DeletionService deletionService = mock(DeletionService.class);
+ int id = 0;
+ String user = "user";
+ String dockerContainerId = "container_e123_12321231_00001";
+ DeletionServiceDeleteTaskProto.Builder protoBuilder =
+ DeletionServiceDeleteTaskProto.newBuilder();
+ protoBuilder
+ .setId(id)
+ .setUser(user)
+ .setDockerContainerId(dockerContainerId);
+ DeletionServiceDeleteTaskProto proto = protoBuilder.build();
+ DeletionTask deletionTask =
+ NMProtoUtils.convertProtoToDockerContainerDeletionTask(proto,
+ deletionService, id);
+ assertEquals(DeletionTaskType.DOCKER_CONTAINER.name(),
+ deletionTask.getDeletionTaskType().name());
+ assertEquals(id, deletionTask.getTaskId());
+ assertEquals(dockerContainerId,
+ ((DockerContainerDeletionTask) deletionTask).getContainerId());
+ }
+
+ @Test
public void testConvertProtoToDeletionTaskRecoveryInfo() throws Exception {
long delTime = System.currentTimeMillis();
List<Integer> successorTaskIds = Arrays.asList(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d65dbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
index 64e6cf0..c7094a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
@@ -25,6 +25,7 @@ import static org.mockito.Matchers.any;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.atLeastOnce;
@@ -73,12 +74,14 @@ import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.ContainerStateTransitionListener;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DockerContainerDeletionMatcher;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEvent;
@@ -94,7 +97,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.Contai
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEventType;
-
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerScheduler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEventType;
@@ -210,6 +213,37 @@ public class TestContainer {
@Test
@SuppressWarnings("unchecked") // mocked generic
+ public void testDockerContainerExternalKill() throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(13, 314159265358979L, 4344, "yak");
+ wc.setupDockerContainerEnv();
+ wc.initContainer();
+ wc.localizeResources();
+ int running = metrics.getRunningContainers();
+ wc.launchContainer();
+ assertEquals(running + 1, metrics.getRunningContainers());
+ reset(wc.localizerBus);
+ wc.containerKilledOnRequest();
+ assertEquals(ContainerState.EXITED_WITH_FAILURE,
+ wc.c.getContainerState());
+ assertNull(wc.c.getLocalizedResources());
+ verifyCleanupCall(wc);
+ int failed = metrics.getFailedContainers();
+ wc.dockerContainerResourcesCleanup();
+ assertEquals(ContainerState.DONE, wc.c.getContainerState());
+ assertEquals(failed + 1, metrics.getFailedContainers());
+ assertEquals(running, metrics.getRunningContainers());
+ }
+ finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked") // mocked generic
public void testContainerPauseAndResume() throws Exception {
WrappedContainer wc = null;
try {
@@ -266,6 +300,30 @@ public class TestContainer {
}
}
}
+
+ @Test
+ @SuppressWarnings("unchecked") // mocked generic
+ public void testDockerContainerCleanupOnFailure() throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(10, 314159265358979L, 4344, "yak");
+ wc.setupDockerContainerEnv();
+ wc.initContainer();
+ wc.localizeResources();
+ wc.launchContainer();
+ reset(wc.localizerBus);
+ wc.containerFailed(ExitCode.FORCE_KILLED.getExitCode());
+ assertEquals(ContainerState.EXITED_WITH_FAILURE,
+ wc.c.getContainerState());
+ assertNull(wc.c.getLocalizedResources());
+ verifyCleanupCall(wc);
+ wc.dockerContainerResourcesCleanup();
+ } finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
@Test
@SuppressWarnings("unchecked") // mocked generic
@@ -322,6 +380,36 @@ public class TestContainer {
@Test
@SuppressWarnings("unchecked") // mocked generic
+ public void testDockerContainerCleanupOnSuccess() throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(11, 314159265358979L, 4344, "yak");
+ wc.setupDockerContainerEnv();
+ wc.initContainer();
+ wc.localizeResources();
+ int running = metrics.getRunningContainers();
+ wc.launchContainer();
+ assertEquals(running + 1, metrics.getRunningContainers());
+ reset(wc.localizerBus);
+ wc.containerSuccessful();
+ assertEquals(ContainerState.EXITED_WITH_SUCCESS,
+ wc.c.getContainerState());
+ assertNull(wc.c.getLocalizedResources());
+ verifyCleanupCall(wc);
+ int completed = metrics.getCompletedContainers();
+ wc.dockerContainerResourcesCleanup();
+ assertEquals(ContainerState.DONE, wc.c.getContainerState());
+ assertEquals(completed + 1, metrics.getCompletedContainers());
+ assertEquals(running, metrics.getRunningContainers());
+ } finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked") // mocked generic
public void testInitWhileDone() throws Exception {
WrappedContainer wc = null;
try {
@@ -341,8 +429,36 @@ public class TestContainer {
assertEquals(ContainerState.DONE, wc.c.getContainerState());
assertNull(wc.c.getLocalizedResources());
verifyCleanupCall(wc);
+ } finally {
+ if (wc != null) {
+ wc.finished();
+ }
}
- finally {
+ }
+
+ @Test
+ @SuppressWarnings("unchecked") // mocked generic
+ public void testDockerContainerInitWhileDone() throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(6, 314159265358979L, 4344, "yak");
+ wc.setupDockerContainerEnv();
+ wc.initContainer();
+ wc.localizeResources();
+ wc.launchContainer();
+ reset(wc.localizerBus);
+ wc.containerSuccessful();
+ wc.dockerContainerResourcesCleanup();
+ assertEquals(ContainerState.DONE, wc.c.getContainerState());
+ verifyOutofBandHeartBeat(wc);
+ assertNull(wc.c.getLocalizedResources());
+ // Now in DONE, issue INIT
+ wc.initContainer();
+ // Verify still in DONE
+ assertEquals(ContainerState.DONE, wc.c.getContainerState());
+ assertNull(wc.c.getLocalizedResources());
+ verifyCleanupCall(wc);
+ } finally {
if (wc != null) {
wc.finished();
}
@@ -377,6 +493,36 @@ public class TestContainer {
}
}
}
+
+ @Test
+ @SuppressWarnings("unchecked")
+ // mocked generic
+ public void testDockerContainerLocalizationFailureAtDone() throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(6, 314159265358979L, 4344, "yak");
+ wc.setupDockerContainerEnv();
+ wc.initContainer();
+ wc.localizeResources();
+ wc.launchContainer();
+ reset(wc.localizerBus);
+ wc.containerSuccessful();
+ wc.dockerContainerResourcesCleanup();
+ assertEquals(ContainerState.DONE, wc.c.getContainerState());
+ verifyOutofBandHeartBeat(wc);
+ assertNull(wc.c.getLocalizedResources());
+ // Now in DONE, issue RESOURCE_FAILED as done by LocalizeRunner
+ wc.resourceFailedContainer();
+ // Verify still in DONE
+ assertEquals(ContainerState.DONE, wc.c.getContainerState());
+ assertNull(wc.c.getLocalizedResources());
+ verifyCleanupCall(wc);
+ } finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
@Test
@SuppressWarnings("unchecked") // mocked generic
@@ -517,6 +663,38 @@ public class TestContainer {
}
@Test
+ public void testDockerKillOnLocalizedWhenContainerNotLaunchedContainerKilled()
+ throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(17, 314159265358979L, 4344, "yak");
+ wc.setupDockerContainerEnv();
+ wc.initContainer();
+ wc.localizeResources();
+ assertEquals(ContainerState.SCHEDULED, wc.c.getContainerState());
+ ContainerLaunch launcher = wc.launcher.running.get(wc.c.getContainerId());
+ wc.killContainer();
+ assertEquals(ContainerState.KILLING, wc.c.getContainerState());
+ launcher.call();
+ wc.drainDispatcherEvents();
+ assertEquals(ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
+ wc.c.getContainerState());
+ assertNull(wc.c.getLocalizedResources());
+ verifyDockerContainerCleanupCall(wc);
+ int killed = metrics.getKilledContainers();
+ wc.c.handle(new ContainerEvent(wc.c.getContainerId(),
+ ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
+ assertEquals(ContainerState.DONE, wc.c.getContainerState());
+ assertEquals(killed + 1, metrics.getKilledContainers());
+ assertEquals(0, metrics.getRunningContainers());
+ } finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
+
+ @Test
public void testKillOnLocalizedWhenContainerNotLaunchedContainerSuccess()
throws Exception {
WrappedContainer wc = null;
@@ -573,6 +751,35 @@ public class TestContainer {
}
@Test
+ public void testDockerKillOnLocalizedContainerNotLaunchedContainerFailure()
+ throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(17, 314159265358979L, 4344, "yak");
+ wc.setupDockerContainerEnv();
+ wc.initContainer();
+ wc.localizeResources();
+ assertEquals(ContainerState.SCHEDULED, wc.c.getContainerState());
+ wc.killContainer();
+ assertEquals(ContainerState.KILLING, wc.c.getContainerState());
+ wc.containerFailed(ExitCode.FORCE_KILLED.getExitCode());
+ wc.drainDispatcherEvents();
+ assertEquals(ContainerState.EXITED_WITH_FAILURE,
+ wc.c.getContainerState());
+ assertNull(wc.c.getLocalizedResources());
+ verifyDockerContainerCleanupCall(wc);
+ wc.c.handle(new ContainerEvent(wc.c.getContainerId(),
+ ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
+ assertEquals(ContainerState.DONE, wc.c.getContainerState());
+ assertEquals(0, metrics.getRunningContainers());
+ } finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
+
+ @Test
public void testKillOnLocalizedWhenContainerLaunched() throws Exception {
WrappedContainer wc = null;
try {
@@ -596,6 +803,33 @@ public class TestContainer {
}
}
}
+
+ @Test
+ public void testDockerKillOnLocalizedWhenContainerLaunched()
+ throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(17, 314159265358979L, 4344, "yak");
+ wc.setupDockerContainerEnv();
+ wc.initContainer();
+ wc.localizeResources();
+ assertEquals(ContainerState.SCHEDULED, wc.c.getContainerState());
+ ContainerLaunch launcher = wc.launcher.running.get(wc.c.getContainerId());
+ launcher.call();
+ wc.drainDispatcherEvents();
+ assertEquals(ContainerState.EXITED_WITH_FAILURE,
+ wc.c.getContainerState());
+ wc.killContainer();
+ assertEquals(ContainerState.EXITED_WITH_FAILURE,
+ wc.c.getContainerState());
+ assertNull(wc.c.getLocalizedResources());
+ verifyDockerContainerCleanupCall(wc);
+ } finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
@Test
public void testResourceLocalizedOnLocalizationFailed() throws Exception {
@@ -733,6 +967,29 @@ public class TestContainer {
}
}
}
+
+ @Test
+ public void testDockerContainerLaunchAfterKillRequest() throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(14, 314159265358979L, 4344, "yak");
+ wc.setupDockerContainerEnv();
+ wc.initContainer();
+ wc.localizeResources();
+ wc.killContainer();
+ assertEquals(ContainerState.KILLING, wc.c.getContainerState());
+ assertNull(wc.c.getLocalizedResources());
+ wc.launchContainer();
+ assertEquals(ContainerState.KILLING, wc.c.getContainerState());
+ assertNull(wc.c.getLocalizedResources());
+ wc.containerKilledOnRequest();
+ verifyDockerContainerCleanupCall(wc);
+ } finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
@Test
public void testContainerRetry() throws Exception{
@@ -843,6 +1100,14 @@ public class TestContainer {
verify(wc.context.getNodeStatusUpdater()).sendOutofBandHeartBeat();
}
+ private void verifyDockerContainerCleanupCall(WrappedContainer wc)
+ throws Exception {
+ DeletionService delService = wc.context.getDeletionService();
+ verify(delService, times(1)).delete(argThat(
+ new DockerContainerDeletionMatcher(delService,
+ wc.c.getContainerId().toString())));
+ }
+
private static class ResourcesReleasedMatcher extends
ArgumentMatcher<LocalizationEvent> {
final HashSet<LocalResourceRequest> resources =
@@ -971,6 +1236,7 @@ public class TestContainer {
final Map<String, LocalResource> localResources;
final Map<String, ByteBuffer> serviceData;
final Context context = mock(Context.class);
+ private final DeletionService delService;
private final Map<ContainerState, ContainerEventType> initStateToEvent =
new HashMap<>();
private final Map<ContainerEventType, ContainerState> eventToFinalState =
@@ -1004,6 +1270,7 @@ public class TestContainer {
auxBus = mock(EventHandler.class);
appBus = mock(EventHandler.class);
LogBus = mock(EventHandler.class);
+ delService = mock(DeletionService.class);
schedBus = new ContainerScheduler(context, dispatcher, metrics, 0) {
@Override
protected void scheduleContainer(Container container) {
@@ -1081,6 +1348,7 @@ public class TestContainer {
}
when(ctxt.getServiceData()).thenReturn(serviceData);
when(ctxt.getContainerRetryContext()).thenReturn(containerRetryContext);
+ when(context.getDeletionService()).thenReturn(delService);
ContainerStateTransitionListener listener =
new ContainerStateTransitionListener() {
@Override
@@ -1213,6 +1481,20 @@ public class TestContainer {
drainDispatcherEvents();
}
+ public void dockerContainerResourcesCleanup() {
+ c.handle(new ContainerEvent(cId,
+ ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
+ verify(delService, times(1)).delete(argThat(
+ new DockerContainerDeletionMatcher(delService, cId.toString())));
+ drainDispatcherEvents();
+ }
+
+ public void setupDockerContainerEnv() {
+ Map<String, String> env = new HashMap<>();
+ env.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker");
+ when(this.ctxt.getEnvironment()).thenReturn(env);
+ }
+
public void containerFailed(int exitCode) {
String diagnosticMsg = "Container completed with exit code " + exitCode;
c.handle(new ContainerExitEvent(cId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d65dbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/DockerContainerDeletionMatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/DockerContainerDeletionMatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/DockerContainerDeletionMatcher.java
new file mode 100644
index 0000000..8fa56fc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/DockerContainerDeletionMatcher.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task;
+
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.mockito.ArgumentMatcher;
+
+/**
+ * ArgumentMatcher to check the arguments of the
+ * {@link DockerContainerDeletionTask}.
+ */
+public class DockerContainerDeletionMatcher
+ extends ArgumentMatcher<DockerContainerDeletionTask> {
+
+ private final DeletionService delService;
+ private final String containerId;
+
+ public DockerContainerDeletionMatcher(DeletionService delService,
+ String containerId) {
+ this.delService = delService;
+ this.containerId = containerId;
+ }
+
+ @Override
+ public boolean matches(Object o) {
+ DockerContainerDeletionTask task = (DockerContainerDeletionTask)o;
+ if (task.getContainerId() == null && containerId == null) {
+ return true;
+ }
+ if (task.getContainerId() != null && containerId != null) {
+ return task.getContainerId().equals(containerId);
+ }
+ return false;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d65dbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/TestDockerContainerDeletionTask.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/TestDockerContainerDeletionTask.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/TestDockerContainerDeletionTask.java
new file mode 100644
index 0000000..738a1dd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/TestDockerContainerDeletionTask.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task;
+
+import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Test the attributes of the {@link DockerContainerDeletionTask} class.
+ */
+public class TestDockerContainerDeletionTask {
+
+ private static final int ID = 0;
+ private static final String USER = "user";
+ private static final String CONTAINER_ID = "container_e123_123456_000001";
+
+ private DeletionService deletionService;
+ private DockerContainerDeletionTask deletionTask;
+
+ @Before
+ public void setUp() throws Exception {
+ deletionService = mock(DeletionService.class);
+ deletionTask = new DockerContainerDeletionTask(ID, deletionService, USER,
+ CONTAINER_ID);
+ }
+
+ @Test
+ public void testGetUser() {
+ assertEquals(USER, deletionTask.getUser());
+ }
+
+ @Test
+ public void testGetContainerId() {
+ assertEquals(CONTAINER_ID, deletionTask.getContainerId());
+ }
+
+ @Test
+ public void testConvertDeletionTaskToProto() {
+ YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto proto =
+ deletionTask.convertDeletionTaskToProto();
+ assertEquals(ID, proto.getId());
+ assertEquals(USER, proto.getUser());
+ assertEquals(CONTAINER_ID, proto.getDockerContainerId());
+ assertEquals(DeletionTaskType.DOCKER_CONTAINER.name(), proto.getTaskType());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d65dbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 4d32427..208455f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -38,7 +38,11 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerKillCommand;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRmCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerStopCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerVolumeCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.DockerCommandPlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
@@ -89,12 +93,12 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.SIGNAL;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER_LOCAL_DIRS;
-import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyBoolean;
import static org.mockito.Mockito.anyList;
import static org.mockito.Mockito.anyMap;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -242,7 +246,7 @@ public class TestDockerContainerRuntime {
// warning annotation on the entire method
verify(mockExecutor, times(1))
.executePrivilegedOperation(anyList(), opCaptor.capture(), any(
- File.class), eq(null), eq(false), eq(false));
+ File.class), anyMap(), anyBoolean(), anyBoolean());
//verification completed. we need to isolate specific invications.
// hence, reset mock here
@@ -1168,11 +1172,12 @@ public class TestDockerContainerRuntime {
}
@Test
- public void testDockerStopOnTermSignal()
+ public void testDockerStopOnTermSignalWhenRunning()
throws ContainerExecutionException, PrivilegedOperationException,
IOException {
List<String> dockerCommands = getDockerCommandsForSignal(
- ContainerExecutor.Signal.TERM);
+ ContainerExecutor.Signal.TERM,
+ DockerCommandExecutor.DockerContainerStatus.RUNNING);
Assert.assertEquals(3, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
Assert.assertEquals(" docker-command=stop", dockerCommands.get(1));
@@ -1180,11 +1185,12 @@ public class TestDockerContainerRuntime {
}
@Test
- public void testDockerStopOnKillSignal()
+ public void testDockerStopOnKillSignalWhenRunning()
throws ContainerExecutionException, PrivilegedOperationException,
IOException {
List<String> dockerCommands = getDockerCommandsForSignal(
- ContainerExecutor.Signal.KILL);
+ ContainerExecutor.Signal.KILL,
+ DockerCommandExecutor.DockerContainerStatus.RUNNING);
Assert.assertEquals(3, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
Assert.assertEquals(" docker-command=stop", dockerCommands.get(1));
@@ -1192,24 +1198,57 @@ public class TestDockerContainerRuntime {
}
@Test
- public void testDockerStopOnQuitSignal()
- throws ContainerExecutionException, PrivilegedOperationException,
- IOException {
+ public void testDockerKillOnQuitSignalWhenRunning() throws Exception {
List<String> dockerCommands = getDockerCommandsForSignal(
- ContainerExecutor.Signal.QUIT);
- Assert.assertEquals(3, dockerCommands.size());
+ ContainerExecutor.Signal.QUIT,
+ DockerCommandExecutor.DockerContainerStatus.RUNNING);
+ Assert.assertEquals(4, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
- Assert.assertEquals(" docker-command=stop", dockerCommands.get(1));
+ Assert.assertEquals(" docker-command=kill", dockerCommands.get(1));
Assert.assertEquals(" name=container_id", dockerCommands.get(2));
+ Assert.assertEquals(" signal=QUIT", dockerCommands.get(3));
+ }
+
+ @Test
+ public void testDockerRmOnWhenExited() throws Exception {
+ env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_DELAYED_REMOVAL,
+ "false");
+ conf.set(YarnConfiguration.NM_DOCKER_ALLOW_DELAYED_REMOVAL, "true");
+ MockRuntime runtime = new MockRuntime(mockExecutor,
+ DockerCommandExecutor.DockerContainerStatus.EXITED, true);
+ builder.setExecutionAttribute(RUN_AS_USER, runAsUser)
+ .setExecutionAttribute(USER, user);
+ runtime.initialize(enableMockContainerExecutor(conf), null);
+ runtime.reapContainer(builder.build());
+ verify(mockExecutor, times(1))
+ .executePrivilegedOperation(anyList(), any(), any(
+ File.class), anyMap(), anyBoolean(), anyBoolean());
+ }
+
+ @Test
+ public void testNoDockerRmWhenDelayedDeletionEnabled()
+ throws Exception {
+ env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_DELAYED_REMOVAL,
+ "true");
+ conf.set(YarnConfiguration.NM_DOCKER_ALLOW_DELAYED_REMOVAL, "true");
+ MockRuntime runtime = new MockRuntime(mockExecutor,
+ DockerCommandExecutor.DockerContainerStatus.EXITED, true);
+ builder.setExecutionAttribute(RUN_AS_USER, runAsUser)
+ .setExecutionAttribute(USER, user);
+ runtime.initialize(enableMockContainerExecutor(conf), null);
+ runtime.reapContainer(builder.build());
+ verify(mockExecutor, never())
+ .executePrivilegedOperation(anyList(), any(), any(
+ File.class), anyMap(), anyBoolean(), anyBoolean());
}
private List<String> getDockerCommandsForSignal(
- ContainerExecutor.Signal signal)
+ ContainerExecutor.Signal signal,
+ DockerCommandExecutor.DockerContainerStatus status)
throws ContainerExecutionException, PrivilegedOperationException,
IOException {
- DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
- mockExecutor, mockCGroupsHandler);
+ MockRuntime runtime = new MockRuntime(mockExecutor, status, false);
builder.setExecutionAttribute(RUN_AS_USER, runAsUser)
.setExecutionAttribute(USER, user)
.setExecutionAttribute(PID, signalPid)
@@ -1576,4 +1615,70 @@ public class TestDockerContainerRuntime {
Assert.assertEquals("CHOWN", it.next());
Assert.assertEquals("DAC_OVERRIDE", it.next());
}
+
+ class MockRuntime extends DockerLinuxContainerRuntime {
+
+ private PrivilegedOperationExecutor privilegedOperationExecutor;
+ private DockerCommandExecutor.DockerContainerStatus containerStatus;
+ private boolean delayedRemovalAllowed;
+
+ MockRuntime(PrivilegedOperationExecutor privilegedOperationExecutor,
+ DockerCommandExecutor.DockerContainerStatus containerStatus,
+ boolean delayedRemovalAllowed) {
+ super(privilegedOperationExecutor);
+ this.privilegedOperationExecutor = privilegedOperationExecutor;
+ this.containerStatus = containerStatus;
+ this.delayedRemovalAllowed = delayedRemovalAllowed;
+ }
+
+ @Override
+ public void signalContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ ContainerExecutor.Signal signal = ctx.getExecutionAttribute(SIGNAL);
+ String containerName = ctx.getContainer().getContainerId().toString();
+ Map<String, String> environment =
+ ctx.getContainer().getLaunchContext().getEnvironment();
+ try {
+ if (ContainerExecutor.Signal.KILL.equals(signal)
+ || ContainerExecutor.Signal.TERM.equals(signal)) {
+ if (DockerCommandExecutor.isStoppable(containerStatus)) {
+ DockerStopCommand dockerStopCommand =
+ new DockerStopCommand(containerName);
+ DockerCommandExecutor.executeDockerCommand(dockerStopCommand,
+ containerName, environment, conf, mockExecutor, false);
+ }
+ } else {
+ if (DockerCommandExecutor.isKillable(containerStatus)) {
+ DockerKillCommand dockerKillCommand =
+ new DockerKillCommand(containerName);
+ dockerKillCommand.setSignal(signal.name());
+ DockerCommandExecutor.executeDockerCommand(dockerKillCommand,
+ containerName, environment, conf, mockExecutor, false);
+ }
+ }
+ } catch (ContainerExecutionException e) {
+ LOG.warn("Signal docker container failed. Exception: ", e);
+ throw new ContainerExecutionException("Signal docker container failed",
+ e.getExitCode(), e.getOutput(), e.getErrorOutput());
+ }
+ }
+
+ @Override
+ public void reapContainer(ContainerRuntimeContext ctx)
+ throws ContainerExecutionException {
+ String delayedRemoval = env.get(ENV_DOCKER_CONTAINER_DELAYED_REMOVAL);
+ if (delayedRemovalAllowed && delayedRemoval != null
+ && delayedRemoval.equalsIgnoreCase("true")) {
+ LOG.info("Delayed removal requested and allowed, skipping removal - "
+ + containerId);
+ } else {
+ if (DockerCommandExecutor.isRemovable(containerStatus)) {
+ DockerRmCommand dockerRmCommand = new DockerRmCommand(containerId);
+ DockerCommandExecutor
+ .executeDockerCommand(dockerRmCommand, containerId, env, conf,
+ privilegedOperationExecutor, false);
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d65dbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
index c362787..94da90b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.MockPrivilegedOperationCaptor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
@@ -42,6 +43,8 @@ import java.util.List;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor.DockerContainerStatus;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
@@ -93,9 +96,8 @@ public class TestDockerCommandExecutor {
public void testExecuteDockerCommand() throws Exception {
DockerStopCommand dockerStopCommand =
new DockerStopCommand(MOCK_CONTAINER_ID);
- DockerCommandExecutor
- .executeDockerCommand(dockerStopCommand, cId.toString(), env,
- configuration, mockExecutor, false);
+ DockerCommandExecutor.executeDockerCommand(dockerStopCommand,
+ cId.toString(), env, configuration, mockExecutor, false);
List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
.capturePrivilegedOperations(mockExecutor, 1, true);
assertEquals(1, ops.size());
@@ -106,9 +108,8 @@ public class TestDockerCommandExecutor {
@Test
public void testExecuteDockerRm() throws Exception {
DockerRmCommand dockerCommand = new DockerRmCommand(MOCK_CONTAINER_ID);
- DockerCommandExecutor
- .executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID, env,
- configuration, mockExecutor, false);
+ DockerCommandExecutor.executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID,
+ env, configuration, mockExecutor, false);
List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
.capturePrivilegedOperations(mockExecutor, 1, true);
List<String> dockerCommands = getValidatedDockerCommands(ops);
@@ -124,9 +125,8 @@ public class TestDockerCommandExecutor {
@Test
public void testExecuteDockerStop() throws Exception {
DockerStopCommand dockerCommand = new DockerStopCommand(MOCK_CONTAINER_ID);
- DockerCommandExecutor
- .executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID, env,
- configuration, mockExecutor, false);
+ DockerCommandExecutor.executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID,
+ env, configuration, mockExecutor, false);
List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
.capturePrivilegedOperations(mockExecutor, 1, true);
List<String> dockerCommands = getValidatedDockerCommands(ops);
@@ -143,9 +143,8 @@ public class TestDockerCommandExecutor {
public void testExecuteDockerInspectStatus() throws Exception {
DockerInspectCommand dockerCommand =
new DockerInspectCommand(MOCK_CONTAINER_ID).getContainerStatus();
- DockerCommandExecutor
- .executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID, env,
- configuration, mockExecutor, false);
+ DockerCommandExecutor.executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID,
+ env, configuration, mockExecutor, false);
List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
.capturePrivilegedOperations(mockExecutor, 1, true);
List<String> dockerCommands = getValidatedDockerCommands(ops);
@@ -164,9 +163,8 @@ public class TestDockerCommandExecutor {
public void testExecuteDockerPull() throws Exception {
DockerPullCommand dockerCommand =
new DockerPullCommand(MOCK_IMAGE_NAME);
- DockerCommandExecutor
- .executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID, env,
- configuration, mockExecutor, false);
+ DockerCommandExecutor.executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID,
+ env, configuration, mockExecutor, false);
List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
.capturePrivilegedOperations(mockExecutor, 1, true);
List<String> dockerCommands = getValidatedDockerCommands(ops);
@@ -183,9 +181,8 @@ public class TestDockerCommandExecutor {
public void testExecuteDockerLoad() throws Exception {
DockerLoadCommand dockerCommand =
new DockerLoadCommand(MOCK_LOCAL_IMAGE_NAME);
- DockerCommandExecutor
- .executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID, env,
- configuration, mockExecutor, false);
+ DockerCommandExecutor.executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID,
+ env, configuration, mockExecutor, false);
List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
.capturePrivilegedOperations(mockExecutor, 1, true);
List<String> dockerCommands = getValidatedDockerCommands(ops);
@@ -206,11 +203,140 @@ public class TestDockerCommandExecutor {
when(mockExecutor.executePrivilegedOperation(eq(null),
any(PrivilegedOperation.class), eq(null), any(), eq(true), eq(false)))
.thenReturn(status.getName());
- assertEquals(status, DockerCommandExecutor
- .getContainerStatus(MOCK_CONTAINER_ID, configuration, mockExecutor));
+ assertEquals(status, DockerCommandExecutor.getContainerStatus(
+ MOCK_CONTAINER_ID, configuration, mockExecutor));
}
}
+ @Test
+ public void testExecuteDockerKillSIGQUIT() throws Exception {
+ DockerKillCommand dockerKillCommand =
+ new DockerKillCommand(MOCK_CONTAINER_ID)
+ .setSignal(ContainerExecutor.Signal.QUIT.name());
+ DockerCommandExecutor.executeDockerCommand(dockerKillCommand,
+ MOCK_CONTAINER_ID, env, configuration, mockExecutor, false);
+ List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
+ .capturePrivilegedOperations(mockExecutor, 1, true);
+ List<String> dockerCommands = getValidatedDockerCommands(ops);
+ assertEquals(1, ops.size());
+ assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
+ ops.get(0).getOperationType().name());
+ assertEquals(4, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=kill", dockerCommands.get(1));
+ assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(2));
+ assertEquals(" signal=" + ContainerExecutor.Signal.QUIT.name(),
+ dockerCommands.get(3));
+ }
+
+ @Test
+ public void testExecuteDockerKillSIGKILL() throws Exception {
+ DockerKillCommand dockerKillCommand =
+ new DockerKillCommand(MOCK_CONTAINER_ID)
+ .setSignal(ContainerExecutor.Signal.KILL.name());
+ DockerCommandExecutor.executeDockerCommand(dockerKillCommand,
+ MOCK_CONTAINER_ID, env, configuration, mockExecutor, false);
+ List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
+ .capturePrivilegedOperations(mockExecutor, 1, true);
+ List<String> dockerCommands = getValidatedDockerCommands(ops);
+ assertEquals(1, ops.size());
+ assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
+ ops.get(0).getOperationType().name());
+ assertEquals(4, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=kill", dockerCommands.get(1));
+ assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(2));
+ assertEquals(" signal=" + ContainerExecutor.Signal.KILL.name(),
+ dockerCommands.get(3));
+ }
+
+ @Test
+ public void testExecuteDockerKillSIGTERM() throws Exception {
+ DockerKillCommand dockerKillCommand =
+ new DockerKillCommand(MOCK_CONTAINER_ID)
+ .setSignal(ContainerExecutor.Signal.TERM.name());
+ DockerCommandExecutor.executeDockerCommand(dockerKillCommand,
+ MOCK_CONTAINER_ID, env, configuration, mockExecutor, false);
+ List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
+ .capturePrivilegedOperations(mockExecutor, 1, true);
+ List<String> dockerCommands = getValidatedDockerCommands(ops);
+ assertEquals(1, ops.size());
+ assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
+ ops.get(0).getOperationType().name());
+ assertEquals(4, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=kill", dockerCommands.get(1));
+ assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(2));
+ assertEquals(" signal=" + ContainerExecutor.Signal.TERM.name(),
+ dockerCommands.get(3));
+ }
+
+ @Test
+ public void testIsStoppable() {
+ assertTrue(DockerCommandExecutor.isStoppable(
+ DockerContainerStatus.RUNNING));
+ assertTrue(DockerCommandExecutor.isStoppable(
+ DockerContainerStatus.RESTARTING));
+ assertFalse(DockerCommandExecutor.isStoppable(
+ DockerContainerStatus.EXITED));
+ assertFalse(DockerCommandExecutor.isStoppable(
+ DockerContainerStatus.CREATED));
+ assertFalse(DockerCommandExecutor.isStoppable(
+ DockerContainerStatus.DEAD));
+ assertFalse(DockerCommandExecutor.isStoppable(
+ DockerContainerStatus.NONEXISTENT));
+ assertFalse(DockerCommandExecutor.isStoppable(
+ DockerContainerStatus.REMOVING));
+ assertFalse(DockerCommandExecutor.isStoppable(
+ DockerContainerStatus.STOPPED));
+ assertFalse(DockerCommandExecutor.isStoppable(
+ DockerContainerStatus.UNKNOWN));
+ }
+
+ @Test
+ public void testIsKIllable() {
+ assertTrue(DockerCommandExecutor.isKillable(
+ DockerContainerStatus.RUNNING));
+ assertTrue(DockerCommandExecutor.isKillable(
+ DockerContainerStatus.RESTARTING));
+ assertFalse(DockerCommandExecutor.isKillable(
+ DockerContainerStatus.EXITED));
+ assertFalse(DockerCommandExecutor.isKillable(
+ DockerContainerStatus.CREATED));
+ assertFalse(DockerCommandExecutor.isKillable(
+ DockerContainerStatus.DEAD));
+ assertFalse(DockerCommandExecutor.isKillable(
+ DockerContainerStatus.NONEXISTENT));
+ assertFalse(DockerCommandExecutor.isKillable(
+ DockerContainerStatus.REMOVING));
+ assertFalse(DockerCommandExecutor.isKillable(
+ DockerContainerStatus.STOPPED));
+ assertFalse(DockerCommandExecutor.isKillable(
+ DockerContainerStatus.UNKNOWN));
+ }
+
+ @Test
+ public void testIsRemovable() {
+ assertTrue(DockerCommandExecutor.isRemovable(
+ DockerContainerStatus.STOPPED));
+ assertTrue(DockerCommandExecutor.isRemovable(
+ DockerContainerStatus.RESTARTING));
+ assertTrue(DockerCommandExecutor.isRemovable(
+ DockerContainerStatus.EXITED));
+ assertTrue(DockerCommandExecutor.isRemovable(
+ DockerContainerStatus.CREATED));
+ assertTrue(DockerCommandExecutor.isRemovable(
+ DockerContainerStatus.DEAD));
+ assertFalse(DockerCommandExecutor.isRemovable(
+ DockerContainerStatus.NONEXISTENT));
+ assertFalse(DockerCommandExecutor.isRemovable(
+ DockerContainerStatus.REMOVING));
+ assertFalse(DockerCommandExecutor.isRemovable(
+ DockerContainerStatus.UNKNOWN));
+ assertFalse(DockerCommandExecutor.isRemovable(
+ DockerContainerStatus.RUNNING));
+ }
+
private List<String> getValidatedDockerCommands(
List<PrivilegedOperation> ops) throws IOException {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d65dbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerKillCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerKillCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerKillCommand.java
new file mode 100644
index 0000000..cd3de2a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerKillCommand.java
@@ -0,0 +1,61 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.util.StringUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests the docker kill command and its command line arguments.
+ */
+public class TestDockerKillCommand {
+
+ private DockerKillCommand dockerKillCommand;
+
+ private static final String SIGNAL = "SIGUSR2";
+ private static final String CONTAINER_NAME = "foo";
+
+ @Before
+ public void setup() {
+ dockerKillCommand = new DockerKillCommand(CONTAINER_NAME);
+ }
+
+ @Test
+ public void testGetCommandOption() {
+ assertEquals("kill", dockerKillCommand.getCommandOption());
+ }
+
+ @Test
+ public void testSetGracePeriod() {
+ dockerKillCommand.setSignal(SIGNAL);
+ assertEquals("kill", StringUtils.join(",",
+ dockerKillCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerKillCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals("SIGUSR2", StringUtils.join(",",
+ dockerKillCommand.getDockerCommandWithArguments().get("signal")));
+ assertEquals(3, dockerKillCommand.getDockerCommandWithArguments().size());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d65dbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
index 2cca277..d7d826c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl.ProcessTreeInfo;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReapContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
@@ -90,6 +91,11 @@ public class TestContainersMonitorResourceChange {
return true;
}
@Override
+ public boolean reapContainer(ContainerReapContext ctx)
+ throws IOException {
+ return true;
+ }
+ @Override
public void deleteAsUser(DeletionAsUserContext ctx)
throws IOException, InterruptedException {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d65dbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/executor/TestContainerReapContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/executor/TestContainerReapContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/executor/TestContainerReapContext.java
new file mode 100644
index 0000000..2cfe29a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/executor/TestContainerReapContext.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.executor;
+
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Test the attributes of the {@link ContainerReapContext}.
+ */
+public class TestContainerReapContext {
+ private final static String USER = "user";
+
+ private Container container;
+ private ContainerReapContext context;
+
+ @Before
+ public void setUp() {
+ container = mock(Container.class);
+ context = new ContainerReapContext.Builder()
+ .setUser(USER)
+ .setContainer(container)
+ .build();
+ }
+
+ @Test
+ public void getContainer() {
+ assertEquals(container, context.getContainer());
+ }
+
+ @Test
+ public void getUser() {
+ assertEquals(USER, context.getUser());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d65dbe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index 1a50c92..a3e4105 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -291,6 +291,7 @@ environment variables in the application's environment:
| `YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER` | Controls whether the Docker container is a privileged container. In order to use privileged containers, the yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed property must be set to true, and the application owner must appear in the value of the yarn.nodemanager.runtime.linux.docker.privileged-containers.acl property. If this environment variable is set to true, a privileged Docker container will be used if allowed. No other value is allowed, so the environment variable should be left unset rather than setting it to false. |
| `YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as "source:dest", where the source is an absolute path that is not a symlink and that points to a localized resource. Note that as of YARN-5298, localized directories are automatically mounted into the container as volumes. |
| `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as "source:dest:mode" and the mode must be "ro" (read-only) or "rw" (read-write) to specify the type of access being requested. The requested mounts will be validated by container-executor based on the values set in container-executor.cfg for docker.allowed.ro-mounts and docker.allowed.rw-mounts. |
+| `YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL` | Allows a user to request delayed deletion of the Docker container on a per container basis. If true, Docker containers will not be removed until the duration defined by yarn.nodemanager.delete.debug-delay-sec has elapsed. Administrators can disable this feature through the yarn-site property yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed. This feature is disabled by default. When this feature is disabled or set to false, the container will be removed as soon as it exits. |
The first two are required. The remainder can be set as needed. While
controlling the container type through environment variables is somewhat less
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org