You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/05/30 21:01:42 UTC
[15/50] [abbrv] hadoop git commit: YARN-4599. Set OOM control for
memory cgroups. (Miklos Szegedi via Haibo Chen)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9686584f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupElasticMemoryController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupElasticMemoryController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupElasticMemoryController.java
new file mode 100644
index 0000000..118d172
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupElasticMemoryController.java
@@ -0,0 +1,319 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.junit.Test;
+
+import java.io.File;
+import java.nio.charset.Charset;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test for elastic non-strict memory controller based on cgroups.
+ */
+public class TestCGroupElasticMemoryController {
+ private YarnConfiguration conf = new YarnConfiguration();
+ private File script = new File("target/" +
+ TestCGroupElasticMemoryController.class.getName());
+
+ /**
+ * Test that at least one memory type is requested.
+ * @throws YarnException on exception
+ */
+ @Test(expected = YarnException.class)
+ public void testConstructorOff()
+ throws YarnException {
+ CGroupElasticMemoryController controller =
+ new CGroupElasticMemoryController(
+ conf,
+ null,
+ null,
+ false,
+ false,
+ 10000
+ );
+ }
+
+ /**
+ * Test that the OOM logic is pluggable.
+ * @throws YarnException on exception
+ */
+ @Test
+ public void testConstructorHandler()
+ throws YarnException {
+ conf.setClass(YarnConfiguration.NM_ELASTIC_MEMORY_CONTROL_OOM_HANDLER,
+ DummyRunnableWithContext.class, Runnable.class);
+ CGroupsHandler handler = mock(CGroupsHandler.class);
+ when(handler.getPathForCGroup(any(), any())).thenReturn("");
+ CGroupElasticMemoryController controller =
+ new CGroupElasticMemoryController(
+ conf,
+ null,
+ handler,
+ true,
+ false,
+ 10000
+ );
+ }
+
+ /**
+ * Test that the handler is notified about multiple OOM events.
+ * @throws Exception on exception
+ */
+ @Test
+ public void testMultipleOOMEvents() throws Exception {
+ conf.set(YarnConfiguration.NM_ELASTIC_MEMORY_CONTROL_OOM_LISTENER_PATH,
+ script.getAbsolutePath());
+ try {
+ FileUtils.writeStringToFile(script,
+ "#!/bin/bash\nprintf oomevent;printf oomevent;\n",
+ Charset.defaultCharset(), false);
+ assertTrue("Could not set executable",
+ script.setExecutable(true));
+
+ CGroupsHandler cgroups = mock(CGroupsHandler.class);
+ when(cgroups.getPathForCGroup(any(), any())).thenReturn("");
+ when(cgroups.getCGroupParam(any(), any(), any()))
+ .thenReturn("under_oom 0");
+
+ Runnable handler = mock(Runnable.class);
+ doNothing().when(handler).run();
+
+ CGroupElasticMemoryController controller =
+ new CGroupElasticMemoryController(
+ conf,
+ null,
+ cgroups,
+ true,
+ false,
+ 10000,
+ handler
+ );
+ controller.run();
+ verify(handler, times(2)).run();
+ } finally {
+ assertTrue(String.format("Could not clean up script %s",
+ script.getAbsolutePath()), script.delete());
+ }
+ }
+
+ /**
+ * Test the scenario that the controller is stopped before.
+ * the child process starts
+ * @throws Exception one exception
+ */
+ @Test
+ public void testStopBeforeStart() throws Exception {
+ conf.set(YarnConfiguration.NM_ELASTIC_MEMORY_CONTROL_OOM_LISTENER_PATH,
+ script.getAbsolutePath());
+ try {
+ FileUtils.writeStringToFile(script,
+ "#!/bin/bash\nprintf oomevent;printf oomevent;\n",
+ Charset.defaultCharset(), false);
+ assertTrue("Could not set executable",
+ script.setExecutable(true));
+
+ CGroupsHandler cgroups = mock(CGroupsHandler.class);
+ when(cgroups.getPathForCGroup(any(), any())).thenReturn("");
+ when(cgroups.getCGroupParam(any(), any(), any()))
+ .thenReturn("under_oom 0");
+
+ Runnable handler = mock(Runnable.class);
+ doNothing().when(handler).run();
+
+ CGroupElasticMemoryController controller =
+ new CGroupElasticMemoryController(
+ conf,
+ null,
+ cgroups,
+ true,
+ false,
+ 10000,
+ handler
+ );
+ controller.stopListening();
+ controller.run();
+ verify(handler, times(0)).run();
+ } finally {
+ assertTrue(String.format("Could not clean up script %s",
+ script.getAbsolutePath()), script.delete());
+ }
+ }
+
+ /**
+ * Test the edge case that OOM is never resolved.
+ * @throws Exception on exception
+ */
+ @Test(expected = YarnRuntimeException.class)
+ public void testInfiniteOOM() throws Exception {
+ conf.set(YarnConfiguration.NM_ELASTIC_MEMORY_CONTROL_OOM_LISTENER_PATH,
+ script.getAbsolutePath());
+ Runnable handler = mock(Runnable.class);
+ try {
+ FileUtils.writeStringToFile(script,
+ "#!/bin/bash\nprintf oomevent;sleep 1000;\n",
+ Charset.defaultCharset(), false);
+ assertTrue("Could not set executable",
+ script.setExecutable(true));
+
+ CGroupsHandler cgroups = mock(CGroupsHandler.class);
+ when(cgroups.getPathForCGroup(any(), any())).thenReturn("");
+ when(cgroups.getCGroupParam(any(), any(), any()))
+ .thenReturn("under_oom 1");
+
+ doNothing().when(handler).run();
+
+ CGroupElasticMemoryController controller =
+ new CGroupElasticMemoryController(
+ conf,
+ null,
+ cgroups,
+ true,
+ false,
+ 10000,
+ handler
+ );
+ controller.run();
+ } finally {
+ verify(handler, times(1)).run();
+ assertTrue(String.format("Could not clean up script %s",
+ script.getAbsolutePath()), script.delete());
+ }
+ }
+
+ /**
+ * Test the edge case that OOM cannot be resolved due to the lack of
+ * containers.
+ * @throws Exception on exception
+ */
+ @Test(expected = YarnRuntimeException.class)
+ public void testNothingToKill() throws Exception {
+ conf.set(YarnConfiguration.NM_ELASTIC_MEMORY_CONTROL_OOM_LISTENER_PATH,
+ script.getAbsolutePath());
+ Runnable handler = mock(Runnable.class);
+ try {
+ FileUtils.writeStringToFile(script,
+ "#!/bin/bash\nprintf oomevent;sleep 1000;\n",
+ Charset.defaultCharset(), false);
+ assertTrue("Could not set executable",
+ script.setExecutable(true));
+
+ CGroupsHandler cgroups = mock(CGroupsHandler.class);
+ when(cgroups.getPathForCGroup(any(), any())).thenReturn("");
+ when(cgroups.getCGroupParam(any(), any(), any()))
+ .thenReturn("under_oom 1");
+
+ doThrow(new YarnRuntimeException("Expected")).when(handler).run();
+
+ CGroupElasticMemoryController controller =
+ new CGroupElasticMemoryController(
+ conf,
+ null,
+ cgroups,
+ true,
+ false,
+ 10000,
+ handler
+ );
+ controller.run();
+ } finally {
+ verify(handler, times(1)).run();
+ assertTrue(String.format("Could not clean up script %s",
+ script.getAbsolutePath()), script.delete());
+ }
+ }
+
+ /**
+ * Test that node manager can exit listening.
+ * This is done by running a long running listener for 10 seconds.
+ * Then we wait for 2 seconds and stop listening.
+ * @throws Exception exception occurred
+ */
+ @Test
+ public void testNormalExit() throws Exception {
+ conf.set(YarnConfiguration.NM_ELASTIC_MEMORY_CONTROL_OOM_LISTENER_PATH,
+ script.getAbsolutePath());
+ try {
+ FileUtils.writeStringToFile(script,
+ "#!/bin/bash\nsleep 10000;\n",
+ Charset.defaultCharset(), false);
+ assertTrue("Could not set executable",
+ script.setExecutable(true));
+
+ CGroupsHandler cgroups = mock(CGroupsHandler.class);
+ when(cgroups.getPathForCGroup(any(), any())).thenReturn("");
+ when(cgroups.getCGroupParam(any(), any(), any()))
+ .thenReturn("under_oom 0");
+
+ Runnable handler = mock(Runnable.class);
+ doNothing().when(handler).run();
+
+ CGroupElasticMemoryController controller =
+ new CGroupElasticMemoryController(
+ conf,
+ null,
+ cgroups,
+ true,
+ false,
+ 10000,
+ handler
+ );
+ ExecutorService service = Executors.newFixedThreadPool(1);
+ service.submit(() -> {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException ex) {
+ assertTrue("Wait interrupted.", false);
+ }
+ controller.stopListening();
+ });
+ controller.run();
+ } finally {
+ assertTrue(String.format("Could not clean up script %s",
+ script.getAbsolutePath()), script.delete());
+ }
+ }
+
+ /**
+ * Test that DefaultOOMHandler is instantiated correctly in
+ * the elastic constructor.
+ * @throws YarnException Could not set up elastic memory control.
+ */
+ @Test
+ public void testDefaultConstructor() throws YarnException{
+ CGroupsHandler handler = mock(CGroupsHandler.class);
+ when(handler.getPathForCGroup(any(), any())).thenReturn("");
+ CGroupElasticMemoryController controller =
+ new CGroupElasticMemoryController(
+ conf, null, handler, true, false, 10);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9686584f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
index 416b4fd..5c7e233 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
@@ -65,17 +65,15 @@ public class TestCGroupsMemoryResourceHandlerImpl {
conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
try {
cGroupsMemoryResourceHandler.bootstrap(conf);
- Assert.fail("Pmem check should not be allowed to run with cgroups");
} catch(ResourceHandlerException re) {
- // do nothing
+ Assert.fail("Pmem check should be allowed to run with cgroups");
}
conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
try {
cGroupsMemoryResourceHandler.bootstrap(conf);
- Assert.fail("Vmem check should not be allowed to run with cgroups");
} catch(ResourceHandlerException re) {
- // do nothing
+ Assert.fail("Vmem check should be allowed to run with cgroups");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9686584f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestDefaultOOMHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestDefaultOOMHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestDefaultOOMHandler.java
new file mode 100644
index 0000000..60c38fe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestDefaultOOMHandler.java
@@ -0,0 +1,307 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGROUP_FILE_TASKS;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGROUP_PARAM_MEMORY_OOM_CONTROL;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGROUP_PARAM_MEMORY_USAGE_BYTES;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test default out of memory handler.
+ */
+public class TestDefaultOOMHandler {
+
+ /**
+ * Test an OOM situation where no containers are running.
+ */
+ @Test(expected = YarnRuntimeException.class)
+ public void testNoContainers() throws Exception {
+ Context context = mock(Context.class);
+
+ when(context.getContainers()).thenReturn(new ConcurrentHashMap<>());
+
+ CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ "",
+ CGROUP_PARAM_MEMORY_OOM_CONTROL))
+ .thenReturn("under_oom 1").thenReturn("under_oom 0");
+
+ DefaultOOMHandler handler = new DefaultOOMHandler(context, false);
+ handler.setCGroupsHandler(cGroupsHandler);
+
+ handler.run();
+ }
+
+ /**
+ * We have two containers, both out of limit. We should kill the later one.
+ *
+ * @throws Exception exception
+ */
+ @Test
+ public void testBothContainersOOM() throws Exception {
+ ConcurrentHashMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<>(new LinkedHashMap<>());
+
+ Container c1 = mock(Container.class);
+ ContainerId cid1 = createContainerId(1);
+ when(c1.getContainerId()).thenReturn(cid1);
+ when(c1.getResource()).thenReturn(Resource.newInstance(10, 1));
+ when(c1.getContainerStartTime()).thenReturn((long) 1);
+ containers.put(createContainerId(1), c1);
+
+ Container c2 = mock(Container.class);
+ ContainerId cid2 = createContainerId(2);
+ when(c2.getContainerId()).thenReturn(cid2);
+ when(c2.getResource()).thenReturn(Resource.newInstance(10, 1));
+ when(c2.getContainerStartTime()).thenReturn((long) 2);
+ containers.put(cid2, c2);
+
+ CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid1.toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1234").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid1.toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(11));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid1.toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(11));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid2.toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1235").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid2.toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(11));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid2.toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(11));
+
+ ContainerExecutor ex = mock(ContainerExecutor.class);
+
+ runOOMHandler(containers, cGroupsHandler, ex);
+
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1235")
+ .setContainer(c2)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(1)).signalContainer(any());
+ }
+
+ /**
+ * We have two containers, one out of limit. We should kill that one.
+ * This should happen even, if it was started earlier
+ *
+ * @throws Exception exception
+ */
+ @Test
+ public void testOneContainerOOM() throws Exception {
+ ConcurrentHashMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<>(new LinkedHashMap<>());
+
+ Container c1 = mock(Container.class);
+ ContainerId cid1 = createContainerId(1);
+ when(c1.getContainerId()).thenReturn(cid1);
+ when(c1.getResource()).thenReturn(Resource.newInstance(10, 1));
+ when(c1.getContainerStartTime()).thenReturn((long) 2);
+ containers.put(createContainerId(1), c1);
+
+ Container c2 = mock(Container.class);
+ ContainerId cid2 = createContainerId(2);
+ when(c2.getContainerId()).thenReturn(cid2);
+ when(c2.getResource()).thenReturn(Resource.newInstance(10, 1));
+ when(c2.getContainerStartTime()).thenReturn((long) 1);
+ containers.put(cid2, c2);
+
+ CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid1.toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1234").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid1.toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid1.toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid2.toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1235").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid2.toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(11));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid2.toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(11));
+
+ ContainerExecutor ex = mock(ContainerExecutor.class);
+ runOOMHandler(containers, cGroupsHandler, ex);
+
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1235")
+ .setContainer(c2)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(1)).signalContainer(any());
+ }
+
+ /**
+ * We have two containers, neither out of limit. We should kill the later one.
+ *
+ * @throws Exception exception
+ */
+ @Test
+ public void testNoContainerOOM() throws Exception {
+ ConcurrentHashMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<>(new LinkedHashMap<>());
+
+ Container c1 = mock(Container.class);
+ ContainerId cid1 = createContainerId(1);
+ when(c1.getContainerId()).thenReturn(cid1);
+ when(c1.getResource()).thenReturn(Resource.newInstance(10, 1));
+ when(c1.getContainerStartTime()).thenReturn((long) 1);
+ containers.put(createContainerId(1), c1);
+
+ Container c2 = mock(Container.class);
+ ContainerId cid2 = createContainerId(2);
+ when(c2.getContainerId()).thenReturn(cid2);
+ when(c2.getResource()).thenReturn(Resource.newInstance(10, 1));
+ when(c2.getContainerStartTime()).thenReturn((long) 2);
+ containers.put(cid2, c2);
+
+ CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class);
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid1.toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1234").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid1.toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid1.toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid2.toString(), CGROUP_FILE_TASKS))
+ .thenReturn("1235").thenReturn("");
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid2.toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES))
+ .thenReturn(getMB(9));
+ when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY,
+ cid2.toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES))
+ .thenReturn(getMB(9));
+
+ ContainerExecutor ex = mock(ContainerExecutor.class);
+ runOOMHandler(containers, cGroupsHandler, ex);
+
+ verify(ex, times(1)).signalContainer(
+ new ContainerSignalContext.Builder()
+ .setPid("1235")
+ .setContainer(c2)
+ .setSignal(ContainerExecutor.Signal.KILL)
+ .build()
+ );
+ verify(ex, times(1)).signalContainer(any());
+ }
+
+ private void runOOMHandler(
+ ConcurrentHashMap<ContainerId, Container> containers,
+ CGroupsHandler cGroupsHandler, ContainerExecutor ex)
+ throws IOException, ResourceHandlerException {
+ Context context = mock(Context.class);
+ when(context.getContainers()).thenReturn(containers);
+
+ when(ex.signalContainer(any()))
+ .thenAnswer(invocation -> {
+ assertEquals("Wrong pid killed", "1235",
+ ((ContainerSignalContext) invocation.getArguments()[0]).getPid());
+ return true;
+ });
+
+ when(cGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ "",
+ CGROUP_PARAM_MEMORY_OOM_CONTROL))
+ .thenReturn("under_oom 1").thenReturn("under_oom 0");
+
+ when(context.getContainerExecutor()).thenReturn(ex);
+
+ DefaultOOMHandler handler = new DefaultOOMHandler(context, false);
+ handler.setCGroupsHandler(cGroupsHandler);
+
+ handler.run();
+ }
+
+ private class AppId extends ApplicationIdPBImpl {
+ AppId(long clusterTs, int appId) {
+ this.setClusterTimestamp(clusterTs);
+ this.setId(appId);
+ }
+ }
+
+ private ContainerId createContainerId(int id) {
+ ApplicationId applicationId = new AppId(1, 1);
+
+ ApplicationAttemptId applicationAttemptId
+ = mock(ApplicationAttemptId.class);
+ when(applicationAttemptId.getApplicationId()).thenReturn(applicationId);
+ when(applicationAttemptId.getAttemptId()).thenReturn(1);
+
+ ContainerId containerId = mock(ContainerId.class);
+ when(containerId.toString()).thenReturn(Integer.toString(id));
+ when(containerId.getContainerId()).thenReturn(new Long(1));
+
+ return containerId;
+ }
+
+ ContainerTokenIdentifier getToken() {
+ ContainerTokenIdentifier id = mock(ContainerTokenIdentifier.class);
+ when(id.getVersion()).thenReturn(1);
+ return id;
+ }
+
+ String getMB(long mb) {
+ return Long.toString(mb * 1024 * 1024);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9686584f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
index 412b8cd..2882b32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
@@ -94,6 +94,7 @@ public class TestContainersMonitor extends BaseContainerManagerTest {
YarnConfiguration.NM_MON_RESOURCE_CALCULATOR,
LinuxResourceCalculatorPlugin.class, ResourceCalculatorPlugin.class);
conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_MEMORY_RESOURCE_ENFORCED, false);
super.setup();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9686584f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
index c5fdccd..8aee532 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -174,9 +174,10 @@ public class TestContainersMonitorResourceChange {
}
@Test
- public void testContainersResourceChange() throws Exception {
+ public void testContainersResourceChangePolling() throws Exception {
// set container monitor interval to be 20ms
conf.setLong(YarnConfiguration.NM_CONTAINER_MON_INTERVAL_MS, 20L);
+ conf.setBoolean(YarnConfiguration.NM_MEMORY_RESOURCE_ENFORCED, false);
containersMonitor = createContainersMonitor(executor, dispatcher, context);
containersMonitor.init(conf);
containersMonitor.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9686584f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCGroupsMemory.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCGroupsMemory.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCGroupsMemory.md
new file mode 100644
index 0000000..ec93234
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerCGroupsMemory.md
@@ -0,0 +1,133 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Using Memory Control in YARN
+=======================
+
+YARN has multiple features to enforce container memory limits. There are three types of controls in YARN that can be used.
+1. The polling feature monitors periodically measures container memory usage and kills the containers that exceed their limits. This is a legacy feature with some issues notably a delay that may lead to node shutdown.
+2. Strict memory control kills each container that has exceeded its limits. It is using the OOM killer capability of the cgroups Linux kernel feature.
+3. Elastic memory control is also based on cgroups. It allows bursting and starts killing containers only, if the overall system memory usage reaches a limit.
+
+If you use 2. or 3. feature 1. is disabled.
+
+Strict Memory Feature
+---------------------
+
+cgroups can be used to preempt containers in case of out-of-memory events. This feature leverages cgroups to clean up containers with the kernel when this happens. If your container exited with exit code `137`, then ou can verify the cause in `/var/log/messages`.
+
+Elastic Memory Feature
+----------------------
+
+The cgroups kernel feature has the ability to notify the node manager, if the parent cgroup of all containers specified by `yarn.nodemanager.linux-container-executor.cgroups.hierarchy` goes over a memory limit. The YARN feature that uses this ability is called elastic memory control. The benefits are that containers can burst using more memory than they are reserved to. This is allowed as long as we do not exceed the overall memory limit. When the limit is reached the kernel freezes all the containers and notifies the node manager. The node manager chooses a container and preempts it. It continues this step until the node is resumed from the OOM condition.
+
+The Limit for Elastic Memory Control
+---------
+
+The limit is the amount of memory allocated to all the containers on the node. The limit is specified by `yarn.nodemanager.resource.memory-mb` and `yarn.nodemanager.vmem-pmem-ratio`. If these are not set, the limit is set based on the available resources. See `yarn.nodemanager.resource.detect-hardware-capabilities` for details.
+
+The pluggable preemption logic
+------------------------------
+
+The preemption logic specifies which container to preempt in a node wide out-of-memory situation. The default logic is the `DefaultOOMHandler`. It picks the latest container that exceeded its memory limit. In the unlikely case that no such container is found, it preempts the container that was launched most recently. This continues until the OOM condition is resolved. This logic supports bursting, when containers use more memory than they reserved as long as we have memory available. This helps to improve the overall cluster utilization. The logic ensures that as long as a container is within its limit, it won't get preempted. If the container bursts it can be preempted. There is a case that all containers are within their limits but we are out of memory. This can also happen in case of oversubscription. We prefer preemting the latest containers to minimize the cost and value lost. Once preempted, the data in the container is lost.
+
+The default out-of-memory handler can be updated using `yarn.nodemanager.elastic-memory-control.oom-handler`. The class named in this configuration entry has to implement java.lang.Runnable. The `run()` function will be called in a node level out-of-memory situation. The constructor should accept an `NmContext` object.
+
+Physical and virtual memory control
+----------------------------------
+
+In case of Elastic Memory Control, the limit applies to the physical or virtual (rss+swap in cgroups) memory depending on whether `yarn.nodemanager.pmem-check-enabled` or `yarn.nodemanager.vmem-check-enabled` is set.
+
+There is no reason to set them both. If the system runs with swap disabled, both will have the same number. If swap is enabled the virtual memory counter will account for pages in physical memory and on the disk. This is what the application allocated and it has control over. The limit should be applied to the virtual memory in this case. When swapping is enabled, the physical memory is no more than the virtual memory and it is adjusted by the kernel not just by the container. There is no point preempting a container when it exceeds a physical memory limit with swapping. The system will just swap out some memory, when needed.
+
+Virtual memory measurement and swapping
+--------------------------------------------
+
+There is a difference between the virtual memory reported by the container monitor and the virtual memory limit specified in the elastic memory control feature. The container monitor uses `ProcfsBasedProcessTree` by default for measurements that returns values from the `proc` file system. The virtual memory returned is the size of the address space of all the processes in each container. This includes anonymous pages, pages swapped out to disk, mapped files and reserved pages among others. Reserved pages are not backed by either physical or swapped memory. They can be a large part of the virtual memory usage. The reservabe address space was limited on 32 bit processors but it is very large on 64-bit ones making this metric less useful. Some Java Virtual Machines reserve large amounts of pages but they do not actually use it. This will result in gigabytes of virtual memory usage shown. However, this does not mean that anything is wrong with the container.
+
+Because of this you can now use `CGroupsResourceCalculator`. This shows only the sum of the physical memory usage and swapped pages as virtual memory usage excluding the reserved address space. This reflects much better what the application and the container allocated.
+
+In order to enable cgroups based resource calculation set `yarn.nodemanager.resource-calculator.class` to `org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsResourceCalculator`.
+
+Configuration quickstart
+------------------------
+
+The following levels of memory enforcement are available and supported:
+
+Level | Configuration type | Options
+---|---|---
+0 | No memory control | All settings below are false
+1 | Strict Container Memory enforcement through polling | P or V
+2 | Strict Container Memory enforcement through cgroups | CG, C and (P or V)
+3 | Elastic Memory Control through cgroups | CG, E and (P or V)
+
+The symbols above mean that the respective configuration entries are `true`:
+
+P: `yarn.nodemanager.pmem-check-enabled`
+
+V: `yarn.nodemanager.vmem-check-enabled`
+
+C: `yarn.nodemanager.resource.memory.enforced`
+
+E: `yarn.nodemanager.elastic-memory-control.enabled`
+
+cgroups prerequisites
+---------------------
+
+CG: C and E require the following prerequisites:
+1. `yarn.nodemanager.container-executor.class` should be `org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor`.
+2. `yarn.nodemanager.runtime.linux.allowed-runtimes` should at least be `default`.
+3. `yarn.nodemanager.resource.memory.enabled` should be `true`
+
+Configuring no memory control
+-----------------------------
+
+`yarn.nodemanager.pmem-check-enabled` and `yarn.nodemanager.vmem-check-enabled` should be `false`.
+
+`yarn.nodemanager.resource.memory.enforced` should be `false`.
+
+`yarn.nodemanager.elastic-memory-control.enabled` should be `false`.
+
+Configuring strict container memory enforcement with polling without cgroups
+----------------------------------------------------------------
+
+`yarn.nodemanager.pmem-check-enabled` or `yarn.nodemanager.vmem-check-enabled` should be `true`.
+
+`yarn.nodemanager.resource.memory.enforced` should be `false`.
+
+`yarn.nodemanager.elastic-memory-control.enabled` should be `false`.
+
+Configuring strict container memory enforcement with cgroups
+------------------------------------------------------------
+
+Strict memory control preempts containers right away using the OOM killer feature of the kernel, when they reach their physical or virtual memory limits. You need to set the following options on top of the prerequisites above to use strict memory control.
+
+Configure the cgroups prerequisites mentioned above.
+
+`yarn.nodemanager.pmem-check-enabled` or `yarn.nodemanager.vmem-check-enabled` should be `true`. You can set them both. **Currently this is ignored by the code, only physical limits can be selected.**
+
+`yarn.nodemanager.resource.memory.enforced` should be true
+
+Configuring elastic memory resource control
+------------------------------------------
+
+The cgroups based elastic memory control preempts containers only if the overall system memory usage reaches its limit allowing bursting. This feature requires setting the following options on top of the prerequisites.
+
+Configure the cgroups prerequisites mentioned above.
+
+`yarn.nodemanager.elastic-memory-control.enabled` should be `true`.
+
+`yarn.nodemanager.resource.memory.enforced` should be `false`
+
+`yarn.nodemanager.pmem-check-enabled` or `yarn.nodemanager.vmem-check-enabled` should be `true`. If swapping is turned off the former should be set, the latter should be set otherwise.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org