You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ka...@apache.org on 2016/11/11 00:10:48 UTC
[21/50] [abbrv] hadoop git commit: YARN-5356. NodeManager should
communicate physical resource capability to ResourceManager. Contributed by
Inigo Goiri
YARN-5356. NodeManager should communicate physical resource capability to ResourceManager. Contributed by Inigo Goiri
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f93ac07
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f93ac07
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f93ac07
Branch: refs/heads/YARN-4752
Commit: 3f93ac0733058238a2c8f23960c986c71dca0e02
Parents: dbb133c
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Nov 8 22:01:26 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Nov 8 22:01:26 2016 +0000
----------------------------------------------------------------------
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 5 +++
.../yarn/sls/scheduler/RMNodeWrapper.java | 5 +++
.../yarn/util/ResourceCalculatorPlugin.java | 39 ++++++++++++++++++++
.../RegisterNodeManagerRequest.java | 24 ++++++++++++
.../pb/RegisterNodeManagerRequestPBImpl.java | 30 ++++++++++++++-
.../yarn_server_common_service_protos.proto | 1 +
.../hadoop/yarn/TestYarnServerApiClasses.java | 3 ++
.../nodemanager/NodeResourceMonitorImpl.java | 6 +--
.../nodemanager/NodeStatusUpdaterImpl.java | 17 ++++++++-
.../monitor/ContainersMonitorImpl.java | 9 +----
.../resourcemanager/ResourceTrackerService.java | 3 +-
.../server/resourcemanager/rmnode/RMNode.java | 6 +++
.../resourcemanager/rmnode/RMNodeImpl.java | 23 +++++++++++-
.../yarn/server/resourcemanager/MockNodes.java | 17 ++++++---
14 files changed, 166 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f93ac07/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index b18fb4a..8962aba 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -218,6 +218,11 @@ public class NodeInfo {
public Integer getDecommissioningTimeout() {
return null;
}
+
+ @Override
+ public Resource getPhysicalResource() {
+ return null;
+ }
}
public static RMNode newNodeInfo(String rackName, String hostName,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f93ac07/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index 4edc216..d7b159c 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -207,4 +207,9 @@ public class RMNodeWrapper implements RMNode {
public Integer getDecommissioningTimeout() {
return null;
}
+
+ @Override
+ public Resource getPhysicalResource() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f93ac07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
index 7b2ea56..e7e4c8a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.SysInfo;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
/**
* Plugin to calculate resource information on the system.
@@ -195,4 +196,42 @@ public class ResourceCalculatorPlugin extends Configured {
return null;
}
+ /**
+ * Create the ResourceCalculatorPlugin for the containers monitor in the Node
+ * Manager and configure it. If the plugin is not configured, this method
+ * will try and return a memory calculator plugin available for this system.
+ *
+ * @param conf Configure the plugin with this.
+ * @return ResourceCalculatorPlugin or null if ResourceCalculatorPlugin is
+ * not available for current system.
+ */
+ public static ResourceCalculatorPlugin getContainersMonitorPlugin(
+ Configuration conf) {
+ Class<? extends ResourceCalculatorPlugin> clazzNM = conf.getClass(
+ YarnConfiguration.NM_MON_RESOURCE_CALCULATOR, null,
+ ResourceCalculatorPlugin.class);
+ Class<? extends ResourceCalculatorPlugin> clazz = conf.getClass(
+ YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR, clazzNM,
+ ResourceCalculatorPlugin.class);
+ return ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
+ }
+
+ /**
+ * Create the ResourceCalculatorPlugin for the node resource monitor in the
+ * Node Manager and configure it. If the plugin is not configured, this
+ * method will try and return a memory calculator plugin available for this
+ * system.
+ *
+ * @param conf Configure the plugin with this.
+ * @return ResourceCalculatorPlugin or null if ResourceCalculatorPlugin is
+ * not available for current system.
+ */
+ public static ResourceCalculatorPlugin getNodeResourceMonitorPlugin(
+ Configuration conf) {
+ Class<? extends ResourceCalculatorPlugin> clazz = conf.getClass(
+ YarnConfiguration.NM_MON_RESOURCE_CALCULATOR, null,
+ ResourceCalculatorPlugin.class);
+ return ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f93ac07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
index 7798ba9..fc30a80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
@@ -41,6 +41,15 @@ public abstract class RegisterNodeManagerRequest {
int httpPort, Resource resource, String nodeManagerVersionId,
List<NMContainerStatus> containerStatuses,
List<ApplicationId> runningApplications, Set<NodeLabel> nodeLabels) {
+ return newInstance(nodeId, httpPort, resource, nodeManagerVersionId,
+ containerStatuses, runningApplications, nodeLabels, null);
+ }
+
+ public static RegisterNodeManagerRequest newInstance(NodeId nodeId,
+ int httpPort, Resource resource, String nodeManagerVersionId,
+ List<NMContainerStatus> containerStatuses,
+ List<ApplicationId> runningApplications, Set<NodeLabel> nodeLabels,
+ Resource physicalResource) {
RegisterNodeManagerRequest request =
Records.newRecord(RegisterNodeManagerRequest.class);
request.setHttpPort(httpPort);
@@ -50,6 +59,7 @@ public abstract class RegisterNodeManagerRequest {
request.setContainerStatuses(containerStatuses);
request.setRunningApplications(runningApplications);
request.setNodeLabels(nodeLabels);
+ request.setPhysicalResource(physicalResource);
return request;
}
@@ -88,4 +98,18 @@ public abstract class RegisterNodeManagerRequest {
*/
public abstract void setRunningApplications(
List<ApplicationId> runningApplications);
+
+ /**
+ * Get the physical resources in the node to properly estimate resource
+ * utilization.
+ * @return Physical resources in the node.
+ */
+ public abstract Resource getPhysicalResource();
+
+ /**
+ * Set the physical resources in the node to properly estimate resource
+ * utilization.
+ * @param physicalResource Physical resources in the node.
+ */
+ public abstract void setPhysicalResource(Resource physicalResource);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f93ac07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
index 2a1a268..0291e0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
@@ -56,6 +56,9 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
private List<ApplicationId> runningApplications = null;
private Set<NodeLabel> labels = null;
+ /** Physical resources in the node. */
+ private Resource physicalResource = null;
+
public RegisterNodeManagerRequestPBImpl() {
builder = RegisterNodeManagerRequestProto.newBuilder();
}
@@ -93,6 +96,9 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
}
builder.setNodeLabels(newBuilder.build());
}
+ if (this.physicalResource != null) {
+ builder.setPhysicalResource(convertToProtoFormat(this.physicalResource));
+ }
}
private synchronized void addNMContainerStatusesToProto() {
@@ -269,7 +275,29 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
initContainerRecoveryReports();
this.containerStatuses.addAll(containerReports);
}
-
+
+ @Override
+ public synchronized Resource getPhysicalResource() {
+ RegisterNodeManagerRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.physicalResource != null) {
+ return this.physicalResource;
+ }
+ if (!p.hasPhysicalResource()) {
+ return null;
+ }
+ this.physicalResource = convertFromProtoFormat(p.getPhysicalResource());
+ return this.physicalResource;
+ }
+
+ @Override
+ public synchronized void setPhysicalResource(Resource pPhysicalResource) {
+ maybeInitBuilder();
+ if (pPhysicalResource == null) {
+ builder.clearPhysicalResource();
+ }
+ this.physicalResource = pPhysicalResource;
+ }
+
@Override
public int hashCode() {
return getProto().hashCode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f93ac07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index 4350fc5..edb2d9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -63,6 +63,7 @@ message RegisterNodeManagerRequestProto {
repeated NMContainerStatusProto container_statuses = 6;
repeated ApplicationIdProto runningApplications = 7;
optional NodeLabelsProto nodeLabels = 8;
+ optional ResourceProto physicalResource = 9;
}
message RegisterNodeManagerResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f93ac07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
index be350e2..b670c36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
@@ -204,6 +204,7 @@ public class TestYarnServerApiClasses {
resource.setMemorySize(10000);
resource.setVirtualCores(2);
original.setResource(resource);
+ original.setPhysicalResource(resource);
RegisterNodeManagerRequestPBImpl copy = new RegisterNodeManagerRequestPBImpl(
original.getProto());
@@ -211,6 +212,8 @@ public class TestYarnServerApiClasses {
assertEquals(9090, copy.getNodeId().getPort());
assertEquals(10000, copy.getResource().getMemorySize());
assertEquals(2, copy.getResource().getVirtualCores());
+ assertEquals(10000, copy.getPhysicalResource().getMemorySize());
+ assertEquals(2, copy.getPhysicalResource().getVirtualCores());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f93ac07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
index 3a78d87..8ee27de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
@@ -66,12 +66,8 @@ public class NodeResourceMonitorImpl extends AbstractService implements
conf.getLong(YarnConfiguration.NM_RESOURCE_MON_INTERVAL_MS,
YarnConfiguration.DEFAULT_NM_RESOURCE_MON_INTERVAL_MS);
- Class<? extends ResourceCalculatorPlugin> clazz =
- conf.getClass(YarnConfiguration.NM_MON_RESOURCE_CALCULATOR, null,
- ResourceCalculatorPlugin.class);
-
this.resourceCalculatorPlugin =
- ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
+ ResourceCalculatorPlugin.getNodeResourceMonitorPlugin(conf);
LOG.info(" Using ResourceCalculatorPlugin : "
+ this.resourceCalculatorPlugin);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f93ac07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index ec7e1d9..6e50d24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -91,6 +91,7 @@ import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePubli
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
import com.google.common.annotations.VisibleForTesting;
@@ -113,6 +114,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
private long nextHeartBeatInterval;
private ResourceTracker resourceTracker;
private Resource totalResource;
+ private Resource physicalResource;
private int httpPort;
private String nodeManagerVersionId;
private String minimumResourceManagerVersion;
@@ -187,6 +189,19 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
this.totalResource = Resource.newInstance(memoryMb, virtualCores);
metrics.addResource(totalResource);
+
+ // Get actual node physical resources
+ int physicalMemoryMb = memoryMb;
+ int physicalCores = virtualCores;
+ ResourceCalculatorPlugin rcp =
+ ResourceCalculatorPlugin.getNodeResourceMonitorPlugin(conf);
+ if (rcp != null) {
+ physicalMemoryMb = (int) (rcp.getPhysicalMemorySize() / (1024 * 1024));
+ physicalCores = rcp.getNumProcessors();
+ }
+ this.physicalResource =
+ Resource.newInstance(physicalMemoryMb, physicalCores);
+
this.tokenKeepAliveEnabled = isTokenKeepAliveEnabled(conf);
this.tokenRemovalDelayMs =
conf.getInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
@@ -343,7 +358,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
RegisterNodeManagerRequest request =
RegisterNodeManagerRequest.newInstance(nodeId, httpPort, totalResource,
nodeManagerVersionId, containerReports, getRunningApplications(),
- nodeLabels);
+ nodeLabels, physicalResource);
if (containerReports != null) {
LOG.info("Registering with RM using containers :" + containerReports);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f93ac07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 72e99d0..fad2b6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -125,15 +125,8 @@ public class ContainersMonitorImpl extends AbstractService implements
this.conf.getLong(YarnConfiguration.NM_RESOURCE_MON_INTERVAL_MS,
YarnConfiguration.DEFAULT_NM_RESOURCE_MON_INTERVAL_MS));
- Class<? extends ResourceCalculatorPlugin> clazz =
- this.conf.getClass(YarnConfiguration
- .NM_CONTAINER_MON_RESOURCE_CALCULATOR,
- this.conf.getClass(
- YarnConfiguration.NM_MON_RESOURCE_CALCULATOR, null,
- ResourceCalculatorPlugin.class),
- ResourceCalculatorPlugin.class);
this.resourceCalculatorPlugin =
- ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, this.conf);
+ ResourceCalculatorPlugin.getContainersMonitorPlugin(this.conf);
LOG.info(" Using ResourceCalculatorPlugin : "
+ this.resourceCalculatorPlugin);
processTreeClass = this.conf.getClass(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f93ac07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 51fc0bd..40bd610 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -318,6 +318,7 @@ public class ResourceTrackerService extends AbstractService implements
int httpPort = request.getHttpPort();
Resource capability = request.getResource();
String nodeManagerVersion = request.getNMVersion();
+ Resource physicalResource = request.getPhysicalResource();
RegisterNodeManagerResponse response = recordFactory
.newRecordInstance(RegisterNodeManagerResponse.class);
@@ -387,7 +388,7 @@ public class ResourceTrackerService extends AbstractService implements
.getCurrentKey());
RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort,
- resolve(host), capability, nodeManagerVersion);
+ resolve(host), capability, nodeManagerVersion, physicalResource);
RMNode oldNode = this.rmContext.getRMNodes().putIfAbsent(nodeId, rmNode);
if (oldNode == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f93ac07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index 6d95b56..86f8679 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -114,6 +114,12 @@ public interface RMNode {
public ResourceUtilization getNodeUtilization();
/**
+ * the physical resources in the node.
+ * @return the physical resources in the node.
+ */
+ Resource getPhysicalResource();
+
+ /**
* The rack name for this node manager.
* @return the rack name.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f93ac07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 71ebddb..302743a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -133,6 +133,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
/* Resource utilization for the node. */
private ResourceUtilization nodeUtilization;
+ /** Physical resources in the node. */
+ private volatile Resource physicalResource;
+
/* Container Queue Information for the node.. Used by Distributed Scheduler */
private OpportunisticContainersStatus opportunisticContainersStatus;
@@ -353,7 +356,15 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
RMNodeEvent> stateMachine;
public RMNodeImpl(NodeId nodeId, RMContext context, String hostName,
- int cmPort, int httpPort, Node node, Resource capability, String nodeManagerVersion) {
+ int cmPort, int httpPort, Node node, Resource capability,
+ String nodeManagerVersion) {
+ this(nodeId, context, hostName, cmPort, httpPort, node, capability,
+ nodeManagerVersion, null);
+ }
+
+ public RMNodeImpl(NodeId nodeId, RMContext context, String hostName,
+ int cmPort, int httpPort, Node node, Resource capability,
+ String nodeManagerVersion, Resource physResource) {
this.nodeId = nodeId;
this.context = context;
this.hostName = hostName;
@@ -367,6 +378,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
this.lastHealthReportTime = System.currentTimeMillis();
this.nodeManagerVersion = nodeManagerVersion;
this.timeStamp = 0;
+ this.physicalResource = physResource;
this.latestNodeHeartBeatResponse.setResponseId(0);
@@ -527,6 +539,15 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
}
@Override
+ public Resource getPhysicalResource() {
+ return this.physicalResource;
+ }
+
+ public void setPhysicalResource(Resource physicalResource) {
+ this.physicalResource = physicalResource;
+ }
+
+ @Override
public NodeState getState() {
this.readLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f93ac07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 7959e74..91170d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -112,12 +112,13 @@ public class MockNodes {
private Set<String> labels;
private ResourceUtilization containersUtilization;
private ResourceUtilization nodeUtilization;
+ private Resource physicalResource;
public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
Resource perNode, String rackName, String healthReport,
long lastHealthReportTime, int cmdPort, String hostName, NodeState state,
Set<String> labels, ResourceUtilization containersUtilization,
- ResourceUtilization nodeUtilization) {
+ ResourceUtilization nodeUtilization, Resource pPhysicalResource) {
this.nodeId = nodeId;
this.nodeAddr = nodeAddr;
this.httpAddress = httpAddress;
@@ -131,6 +132,7 @@ public class MockNodes {
this.labels = labels;
this.containersUtilization = containersUtilization;
this.nodeUtilization = nodeUtilization;
+ this.physicalResource = pPhysicalResource;
}
@Override
@@ -277,6 +279,11 @@ public class MockNodes {
public Integer getDecommissioningTimeout() {
return null;
}
+
+ @Override
+ public Resource getPhysicalResource() {
+ return this.physicalResource;
+ }
};
private static RMNode buildRMNode(int rack, final Resource perNode,
@@ -287,19 +294,19 @@ public class MockNodes {
private static RMNode buildRMNode(int rack, final Resource perNode,
NodeState state, String httpAddr, Set<String> labels) {
return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123,
- labels, null, null);
+ labels, null, null, null);
}
private static RMNode buildRMNode(int rack, final Resource perNode,
NodeState state, String httpAddr, int hostnum, String hostName, int port) {
return buildRMNode(rack, perNode, state, httpAddr, hostnum, hostName, port,
- null, null, null);
+ null, null, null, null);
}
private static RMNode buildRMNode(int rack, final Resource perNode,
NodeState state, String httpAddr, int hostnum, String hostName, int port,
Set<String> labels, ResourceUtilization containersUtilization,
- ResourceUtilization nodeUtilization) {
+ ResourceUtilization nodeUtilization, Resource physicalResource) {
final String rackName = "rack"+ rack;
final int nid = hostnum;
final String nodeAddr = hostName + ":" + nid;
@@ -312,7 +319,7 @@ public class MockNodes {
String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe";
return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode,
rackName, healthReport, 0, nid, hostName, state, labels,
- containersUtilization, nodeUtilization);
+ containersUtilization, nodeUtilization, physicalResource);
}
public static RMNode nodeInfo(int rack, final Resource perNode,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org