You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2017/08/03 16:57:37 UTC
hadoop git commit: YARN-6673 Add cpu cgroup configurations for
opportunistic containers. (Miklos Szegedi via Haibo Chen)
Repository: hadoop
Updated Branches:
refs/heads/trunk c5d256c76 -> 293c74a81
YARN-6673 Add cpu cgroup configurations for opportunistic containers. (Miklos Szegedi via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/293c74a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/293c74a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/293c74a8
Branch: refs/heads/trunk
Commit: 293c74a81b407274621751e2104738b2a09e666e
Parents: c5d256c
Author: Haibo Chen <ha...@apache.org>
Authored: Thu Aug 3 09:56:51 2017 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Thu Aug 3 09:56:51 2017 -0700
----------------------------------------------------------------------
.../CGroupsCpuResourceHandlerImpl.java | 20 ++++++++++++-----
.../TestCGroupsCpuResourceHandlerImpl.java | 23 ++++++++++++++++++++
2 files changed, 38 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/293c74a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
index d9cca8f..830782d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
@@ -26,8 +26,10 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
@@ -72,6 +74,7 @@ public class CGroupsCpuResourceHandlerImpl implements CpuResourceHandler {
static final int MIN_PERIOD_US = 1000;
@VisibleForTesting
static final int CPU_DEFAULT_WEIGHT = 1024; // set by kernel
+ static final int CPU_DEFAULT_WEIGHT_OPPORTUNISTIC = 2;
CGroupsCpuResourceHandlerImpl(CGroupsHandler cGroupsHandler) {
this.cGroupsHandler = cGroupsHandler;
@@ -181,16 +184,23 @@ public class CGroupsCpuResourceHandlerImpl implements CpuResourceHandler {
@Override
public List<PrivilegedOperation> preStart(Container container)
throws ResourceHandlerException {
-
String cgroupId = container.getContainerId().toString();
Resource containerResource = container.getResource();
cGroupsHandler.createCGroup(CPU, cgroupId);
try {
int containerVCores = containerResource.getVirtualCores();
- int cpuShares = CPU_DEFAULT_WEIGHT * containerVCores;
- cGroupsHandler
- .updateCGroupParam(CPU, cgroupId, CGroupsHandler.CGROUP_CPU_SHARES,
- String.valueOf(cpuShares));
+ ContainerTokenIdentifier id = container.getContainerTokenIdentifier();
+ if (id != null && id.getExecutionType() ==
+ ExecutionType.OPPORTUNISTIC) {
+ cGroupsHandler
+ .updateCGroupParam(CPU, cgroupId, CGroupsHandler.CGROUP_CPU_SHARES,
+ String.valueOf(CPU_DEFAULT_WEIGHT_OPPORTUNISTIC));
+ } else {
+ int cpuShares = CPU_DEFAULT_WEIGHT * containerVCores;
+ cGroupsHandler
+ .updateCGroupParam(CPU, cgroupId, CGroupsHandler.CGROUP_CPU_SHARES,
+ String.valueOf(cpuShares));
+ }
if (strictResourceUsageMode) {
if (nodeVCores != containerVCores) {
float containerCPU =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/293c74a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java
index 674cd71..006b060 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java
@@ -21,8 +21,10 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resourc
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
@@ -294,4 +296,25 @@ public class TestCGroupsCpuResourceHandlerImpl {
public void testStrictResourceUsage() throws Exception {
Assert.assertNull(cGroupsCpuResourceHandler.teardown());
}
+
+ @Test
+ public void testOpportunistic() throws Exception {
+ Configuration conf = new YarnConfiguration();
+
+ cGroupsCpuResourceHandler.bootstrap(plugin, conf);
+ ContainerTokenIdentifier tokenId = mock(ContainerTokenIdentifier.class);
+ when(tokenId.getExecutionType()).thenReturn(ExecutionType.OPPORTUNISTIC);
+ Container container = mock(Container.class);
+ String id = "container_01_01";
+ ContainerId mockContainerId = mock(ContainerId.class);
+ when(mockContainerId.toString()).thenReturn(id);
+ when(container.getContainerId()).thenReturn(mockContainerId);
+ when(container.getContainerTokenIdentifier()).thenReturn(tokenId);
+ when(container.getResource()).thenReturn(Resource.newInstance(1024, 2));
+ cGroupsCpuResourceHandler.preStart(container);
+ verify(mockCGroupsHandler, times(1))
+ .updateCGroupParam(CGroupsHandler.CGroupController.CPU, id,
+ CGroupsHandler.CGROUP_CPU_SHARES, "2");
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org