You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/05/14 17:44:06 UTC
[03/50] [abbrv] hadoop git commit: YARN-7715. Support NM
promotion/demotion of running containers. (Miklos Szegedi via Haibo Chen)
YARN-7715. Support NM promotion/demotion of running containers. (Miklos Szegedi via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3183b35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3183b35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3183b35
Branch: refs/heads/HDDS-4
Commit: d3183b352fb19d03482a00986cc04fad57cd842d
Parents: ee7daf0
Author: Haibo Chen <ha...@apache.org>
Authored: Thu May 10 11:01:01 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Mon May 14 10:31:08 2018 -0700
----------------------------------------------------------------------
.../CGroupsBlkioResourceHandlerImpl.java | 6 ++
.../CGroupsCpuResourceHandlerImpl.java | 75 ++++++++++++--------
.../CGroupsMemoryResourceHandlerImpl.java | 73 +++++++++++--------
.../NetworkPacketTaggingHandlerImpl.java | 6 ++
.../linux/resources/ResourceHandler.java | 16 ++++-
.../linux/resources/ResourceHandlerChain.java | 18 +++++
.../TrafficControlBandwidthHandlerImpl.java | 6 ++
.../resources/fpga/FpgaResourceHandlerImpl.java | 6 ++
.../resources/gpu/GpuResourceHandlerImpl.java | 6 ++
.../resources/numa/NumaResourceHandlerImpl.java | 6 ++
.../scheduler/ContainerScheduler.java | 30 ++++++++
.../TestCGroupsCpuResourceHandlerImpl.java | 1 +
.../TestCGroupsMemoryResourceHandlerImpl.java | 1 +
.../TestResourcePluginManager.java | 6 ++
.../TestContainerSchedulerQueuing.java | 10 +++
15 files changed, 202 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
index 42fc634..2c402c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
@@ -156,6 +156,12 @@ public class CGroupsBlkioResourceHandlerImpl implements DiskResourceHandler {
}
@Override
+ public List<PrivilegedOperation> updateContainer(Container container)
+ throws ResourceHandlerException {
+ return null;
+ }
+
+ @Override
public List<PrivilegedOperation> postComplete(ContainerId containerId)
throws ResourceHandlerException {
cGroupsHandler.deleteCGroup(CGroupsHandler.CGroupController.BLKIO,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
index 7ea7be2..37221f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
@@ -186,38 +186,8 @@ public class CGroupsCpuResourceHandlerImpl implements CpuResourceHandler {
public List<PrivilegedOperation> preStart(Container container)
throws ResourceHandlerException {
String cgroupId = container.getContainerId().toString();
- Resource containerResource = container.getResource();
cGroupsHandler.createCGroup(CPU, cgroupId);
- try {
- int containerVCores = containerResource.getVirtualCores();
- ContainerTokenIdentifier id = container.getContainerTokenIdentifier();
- if (id != null && id.getExecutionType() ==
- ExecutionType.OPPORTUNISTIC) {
- cGroupsHandler
- .updateCGroupParam(CPU, cgroupId, CGroupsHandler.CGROUP_CPU_SHARES,
- String.valueOf(CPU_DEFAULT_WEIGHT_OPPORTUNISTIC));
- } else {
- int cpuShares = CPU_DEFAULT_WEIGHT * containerVCores;
- cGroupsHandler
- .updateCGroupParam(CPU, cgroupId, CGroupsHandler.CGROUP_CPU_SHARES,
- String.valueOf(cpuShares));
- }
- if (strictResourceUsageMode) {
- if (nodeVCores != containerVCores) {
- float containerCPU =
- (containerVCores * yarnProcessors) / (float) nodeVCores;
- int[] limits = getOverallLimits(containerCPU);
- cGroupsHandler.updateCGroupParam(CPU, cgroupId,
- CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf(limits[0]));
- cGroupsHandler.updateCGroupParam(CPU, cgroupId,
- CGroupsHandler.CGROUP_CPU_QUOTA_US, String.valueOf(limits[1]));
- }
- }
- } catch (ResourceHandlerException re) {
- cGroupsHandler.deleteCGroup(CPU, cgroupId);
- LOG.warn("Could not update cgroup for container", re);
- throw re;
- }
+ updateContainer(container);
List<PrivilegedOperation> ret = new ArrayList<>();
ret.add(new PrivilegedOperation(
PrivilegedOperation.OperationType.ADD_PID_TO_CGROUP,
@@ -233,6 +203,49 @@ public class CGroupsCpuResourceHandlerImpl implements CpuResourceHandler {
}
@Override
+ public List<PrivilegedOperation> updateContainer(Container container)
+ throws ResourceHandlerException {
+ Resource containerResource = container.getResource();
+ String cgroupId = container.getContainerId().toString();
+ File cgroup = new File(cGroupsHandler.getPathForCGroup(CPU, cgroupId));
+ if (cgroup.exists()) {
+ try {
+ int containerVCores = containerResource.getVirtualCores();
+ ContainerTokenIdentifier id = container.getContainerTokenIdentifier();
+ if (id != null && id.getExecutionType() ==
+ ExecutionType.OPPORTUNISTIC) {
+ cGroupsHandler
+ .updateCGroupParam(CPU, cgroupId,
+ CGroupsHandler.CGROUP_CPU_SHARES,
+ String.valueOf(CPU_DEFAULT_WEIGHT_OPPORTUNISTIC));
+ } else {
+ int cpuShares = CPU_DEFAULT_WEIGHT * containerVCores;
+ cGroupsHandler
+ .updateCGroupParam(CPU, cgroupId,
+ CGroupsHandler.CGROUP_CPU_SHARES,
+ String.valueOf(cpuShares));
+ }
+ if (strictResourceUsageMode) {
+ if (nodeVCores != containerVCores) {
+ float containerCPU =
+ (containerVCores * yarnProcessors) / (float) nodeVCores;
+ int[] limits = getOverallLimits(containerCPU);
+ cGroupsHandler.updateCGroupParam(CPU, cgroupId,
+ CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf(limits[0]));
+ cGroupsHandler.updateCGroupParam(CPU, cgroupId,
+ CGroupsHandler.CGROUP_CPU_QUOTA_US, String.valueOf(limits[1]));
+ }
+ }
+ } catch (ResourceHandlerException re) {
+ cGroupsHandler.deleteCGroup(CPU, cgroupId);
+ LOG.warn("Could not update cgroup for container", re);
+ throw re;
+ }
+ }
+ return null;
+ }
+
+ @Override
public List<PrivilegedOperation> postComplete(ContainerId containerId)
throws ResourceHandlerException {
cGroupsHandler.deleteCGroup(CPU, containerId.toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
index 558751f..2d1585e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
+import java.io.File;
import java.util.ArrayList;
import java.util.List;
@@ -119,43 +120,53 @@ public class CGroupsMemoryResourceHandlerImpl implements MemoryResourceHandler {
}
@Override
- public List<PrivilegedOperation> preStart(Container container)
+ public List<PrivilegedOperation> updateContainer(Container container)
throws ResourceHandlerException {
-
String cgroupId = container.getContainerId().toString();
- //memory is in MB
- long containerSoftLimit =
- (long) (container.getResource().getMemorySize() * this.softLimit);
- long containerHardLimit = container.getResource().getMemorySize();
- cGroupsHandler.createCGroup(MEMORY, cgroupId);
- if (enforce) {
- try {
- cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
- CGroupsHandler.CGROUP_PARAM_MEMORY_HARD_LIMIT_BYTES,
- String.valueOf(containerHardLimit) + "M");
- ContainerTokenIdentifier id = container.getContainerTokenIdentifier();
- if (id != null && id.getExecutionType() ==
- ExecutionType.OPPORTUNISTIC) {
- cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
- CGroupsHandler.CGROUP_PARAM_MEMORY_SOFT_LIMIT_BYTES,
- String.valueOf(OPPORTUNISTIC_SOFT_LIMIT) + "M");
+ File cgroup = new File(cGroupsHandler.getPathForCGroup(MEMORY, cgroupId));
+ if (cgroup.exists()) {
+ //memory is in MB
+ long containerSoftLimit =
+ (long) (container.getResource().getMemorySize() * this.softLimit);
+ long containerHardLimit = container.getResource().getMemorySize();
+ if (enforce) {
+ try {
cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
- CGroupsHandler.CGROUP_PARAM_MEMORY_SWAPPINESS,
- String.valueOf(OPPORTUNISTIC_SWAPPINESS));
- } else {
- cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
- CGroupsHandler.CGROUP_PARAM_MEMORY_SOFT_LIMIT_BYTES,
- String.valueOf(containerSoftLimit) + "M");
- cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
- CGroupsHandler.CGROUP_PARAM_MEMORY_SWAPPINESS,
- String.valueOf(swappiness));
+ CGroupsHandler.CGROUP_PARAM_MEMORY_HARD_LIMIT_BYTES,
+ String.valueOf(containerHardLimit) + "M");
+ ContainerTokenIdentifier id = container.getContainerTokenIdentifier();
+ if (id != null && id.getExecutionType() ==
+ ExecutionType.OPPORTUNISTIC) {
+ cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
+ CGroupsHandler.CGROUP_PARAM_MEMORY_SOFT_LIMIT_BYTES,
+ String.valueOf(OPPORTUNISTIC_SOFT_LIMIT) + "M");
+ cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
+ CGroupsHandler.CGROUP_PARAM_MEMORY_SWAPPINESS,
+ String.valueOf(OPPORTUNISTIC_SWAPPINESS));
+ } else {
+ cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
+ CGroupsHandler.CGROUP_PARAM_MEMORY_SOFT_LIMIT_BYTES,
+ String.valueOf(containerSoftLimit) + "M");
+ cGroupsHandler.updateCGroupParam(MEMORY, cgroupId,
+ CGroupsHandler.CGROUP_PARAM_MEMORY_SWAPPINESS,
+ String.valueOf(swappiness));
+ }
+ } catch (ResourceHandlerException re) {
+ cGroupsHandler.deleteCGroup(MEMORY, cgroupId);
+ LOG.warn("Could not update cgroup for container", re);
+ throw re;
}
- } catch (ResourceHandlerException re) {
- cGroupsHandler.deleteCGroup(MEMORY, cgroupId);
- LOG.warn("Could not update cgroup for container", re);
- throw re;
}
}
+ return null;
+ }
+
+ @Override
+ public List<PrivilegedOperation> preStart(Container container)
+ throws ResourceHandlerException {
+ String cgroupId = container.getContainerId().toString();
+ cGroupsHandler.createCGroup(MEMORY, cgroupId);
+ updateContainer(container);
List<PrivilegedOperation> ret = new ArrayList<>();
ret.add(new PrivilegedOperation(
PrivilegedOperation.OperationType.ADD_PID_TO_CGROUP,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkPacketTaggingHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkPacketTaggingHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkPacketTaggingHandlerImpl.java
index 1580e2c..3f6d4b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkPacketTaggingHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkPacketTaggingHandlerImpl.java
@@ -128,6 +128,12 @@ public class NetworkPacketTaggingHandlerImpl
return null;
}
+ @Override
+ public List<PrivilegedOperation> updateContainer(Container container)
+ throws ResourceHandlerException {
+ return null;
+ }
+
/**
* Cleanup operation once container is completed - deletes cgroup.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandler.java
index 3dfc86b..35c6460 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandler.java
@@ -61,7 +61,7 @@ public interface ResourceHandler {
/**
* Require state for container that was already launched
*
- * @param containerId if of the container being reacquired.
+ * @param containerId id of the container being reacquired.
* @return (possibly empty) list of operations that require elevated
* privileges
* @throws ResourceHandlerException
@@ -71,7 +71,19 @@ public interface ResourceHandler {
throws ResourceHandlerException;
/**
- * Perform any tasks necessary after container completion
+ * Update state for container that was already launched
+ *
+ * @param container the container being updated.
+ * @return (possibly empty) list of operations that require elevated
+ * privileges
+ * @throws ResourceHandlerException
+ */
+
+ List<PrivilegedOperation> updateContainer(Container container)
+ throws ResourceHandlerException;
+
+ /**
+ * Perform any tasks necessary after container completion.
* @param containerId of the container that was completed.
* @return (possibly empty) list of operations that require elevated
* privileges
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerChain.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerChain.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerChain.java
index 72bf30c..2fc301a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerChain.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerChain.java
@@ -101,6 +101,24 @@ public class ResourceHandlerChain implements ResourceHandler {
}
@Override
+ public List<PrivilegedOperation> updateContainer(Container container)
+ throws ResourceHandlerException {
+ List<PrivilegedOperation> allOperations = new
+ ArrayList<PrivilegedOperation>();
+
+ for (ResourceHandler resourceHandler : resourceHandlers) {
+ List<PrivilegedOperation> handlerOperations =
+ resourceHandler.updateContainer(container);
+
+ if (handlerOperations != null) {
+ allOperations.addAll(handlerOperations);
+ }
+
+ }
+ return allOperations;
+ }
+
+ @Override
public List<PrivilegedOperation> postComplete(ContainerId containerId)
throws ResourceHandlerException {
List<PrivilegedOperation> allOperations = new
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
index 126685f..c04e935 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
@@ -203,6 +203,12 @@ public class TrafficControlBandwidthHandlerImpl
return null;
}
+ @Override
+ public List<PrivilegedOperation> updateContainer(Container container)
+ throws ResourceHandlerException {
+ return null;
+ }
+
/**
* Returns total bytes sent per container to be used for metrics tracking
* purposes.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java
index bf3d9b0..11f7114 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java
@@ -206,6 +206,12 @@ public class FpgaResourceHandlerImpl implements ResourceHandler {
}
@Override
+ public List<PrivilegedOperation> updateContainer(Container container)
+ throws ResourceHandlerException {
+ return null;
+ }
+
+ @Override
public List<PrivilegedOperation> postComplete(ContainerId containerId) throws ResourceHandlerException {
allocator.cleanupAssignFpgas(containerId.toString());
cGroupsHandler.deleteCGroup(CGroupsHandler.CGroupController.DEVICES,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
index 8ddc227..587fcb4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
@@ -165,6 +165,12 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
}
@Override
+ public List<PrivilegedOperation> updateContainer(Container container)
+ throws ResourceHandlerException {
+ return null;
+ }
+
+ @Override
public synchronized List<PrivilegedOperation> postComplete(
ContainerId containerId) throws ResourceHandlerException {
gpuAllocator.cleanupAssignGpus(containerId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceHandlerImpl.java
index 128daca..8ffba24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceHandlerImpl.java
@@ -95,6 +95,12 @@ public class NumaResourceHandlerImpl implements ResourceHandler {
}
@Override
+ public List<PrivilegedOperation> updateContainer(Container container)
+ throws ResourceHandlerException {
+ return null;
+ }
+
+ @Override
public List<PrivilegedOperation> postComplete(ContainerId containerId)
throws ResourceHandlerException {
numaResourceAllocator.releaseNumaResource(containerId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 57368ab..5cdcf41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -34,6 +34,9 @@ import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor
.ChangeMonitoringContainerResourceEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
@@ -105,6 +108,9 @@ public class ContainerScheduler extends AbstractService implements
private Boolean usePauseEventForPreemption = false;
+ @VisibleForTesting
+ ResourceHandlerChain resourceHandlerChain = null;
+
/**
* Instantiate a Container Scheduler.
* @param context NodeManager Context.
@@ -123,6 +129,24 @@ public class ContainerScheduler extends AbstractService implements
@Override
public void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
+ try {
+ if (resourceHandlerChain == null) {
+ resourceHandlerChain = ResourceHandlerModule
+ .getConfiguredResourceHandlerChain(conf, context);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Resource handler chain enabled = " + (resourceHandlerChain
+ != null));
+ }
+ if (resourceHandlerChain != null) {
+ LOG.debug("Bootstrapping resource handler chain");
+ resourceHandlerChain.bootstrap(conf);
+ }
+ } catch (ResourceHandlerException e) {
+ LOG.error("Failed to bootstrap configured resource subsystems! ", e);
+ throw new IOException(
+ "Failed to bootstrap configured resource subsystems!");
+ }
this.usePauseEventForPreemption =
conf.getBoolean(
YarnConfiguration.NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION,
@@ -218,6 +242,12 @@ public class ContainerScheduler extends AbstractService implements
updateEvent.getContainer());
}
}
+ try {
+ resourceHandlerChain.updateContainer(updateEvent.getContainer());
+ } catch (Exception ex) {
+ LOG.warn(String.format("Could not update resources on " +
+ "continer update of %s", containerId), ex);
+ }
startPendingContainers(maxOppQueueLength <= 0);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java
index 006b060..842fc6b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsCpuResourceHandlerImpl.java
@@ -49,6 +49,7 @@ public class TestCGroupsCpuResourceHandlerImpl {
@Before
public void setup() {
mockCGroupsHandler = mock(CGroupsHandler.class);
+ when(mockCGroupsHandler.getPathForCGroup(any(), any())).thenReturn(".");
cGroupsCpuResourceHandler =
new CGroupsCpuResourceHandlerImpl(mockCGroupsHandler);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
index 78ccc61..416b4fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
@@ -45,6 +45,7 @@ public class TestCGroupsMemoryResourceHandlerImpl {
@Before
public void setup() {
mockCGroupsHandler = mock(CGroupsHandler.class);
+ when(mockCGroupsHandler.getPathForCGroup(any(), any())).thenReturn(".");
cGroupsMemoryResourceHandler =
new CGroupsMemoryResourceHandlerImpl(mockCGroupsHandler);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
index bcadf76..6ed7c56 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
@@ -117,6 +117,12 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
}
@Override
+ public List<PrivilegedOperation> updateContainer(Container container)
+ throws ResourceHandlerException {
+ return null;
+ }
+
+ @Override
public List<PrivilegedOperation> postComplete(ContainerId containerId)
throws ResourceHandlerException {
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3183b35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
index 5c72e7e..1da7e4a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
@@ -72,7 +73,11 @@ import org.junit.Assert;
import org.junit.Test;
import org.slf4j.LoggerFactory;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
/**
* Tests to verify that the {@link ContainerScheduler} is able to queue and
@@ -1183,6 +1188,8 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
ContainerScheduler containerScheduler =
containerManager.getContainerScheduler();
+ containerScheduler.resourceHandlerChain =
+ mock(ResourceHandlerChain.class);
// Ensure two containers are properly queued.
Assert.assertEquals(1, containerScheduler.getNumQueuedContainers());
Assert.assertEquals(0,
@@ -1246,6 +1253,9 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
ContainerEventType.INIT_CONTAINER,
ContainerEventType.UPDATE_CONTAINER_TOKEN,
ContainerEventType.CONTAINER_LAUNCHED), containerEventTypes);
+ verify(containerScheduler.resourceHandlerChain,
+ times(1))
+ .updateContainer(any());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org