You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ra...@apache.org on 2016/12/15 09:04:47 UTC
[19/50] [abbrv] hadoop git commit: YARN-5982. Simplify opportunistic
container parameters and metrics. (Konstantinos Karanasos via asuresh)
YARN-5982. Simplify opportunistic container parameters and metrics. (Konstantinos Karanasos via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0aace21
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0aace21
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0aace21
Branch: refs/heads/HDFS-10285
Commit: b0aace21b1ef3436ba9d516186208fee9a9ceef2
Parents: 55f5886
Author: Arun Suresh <as...@apache.org>
Authored: Fri Dec 9 16:41:25 2016 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Fri Dec 9 16:41:25 2016 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 41 ------------
.../src/main/resources/yarn-default.xml | 57 -----------------
.../scheduler/ContainerScheduler.java | 10 +--
.../nodemanager/metrics/NodeManagerMetrics.java | 65 ++++++++++----------
...pportunisticContainerAllocatorAMService.java | 60 ++++--------------
.../scheduler/SchedulerNode.java | 6 +-
.../resourcemanager/webapp/NodesPage.java | 6 +-
.../resourcemanager/webapp/dao/NodeInfo.java | 10 +--
.../webapp/TestRMWebServicesNodes.java | 10 +--
9 files changed, 68 insertions(+), 197 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 4934964..69c7b00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -323,47 +323,6 @@ public class YarnConfiguration extends Configuration {
public static final boolean
OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED_DEFAULT = false;
- /** Minimum memory (in MB) used for allocating an opportunistic container. */
- public static final String OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB =
- YARN_PREFIX + "opportunistic-containers.min-memory-mb";
- public static final int OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB_DEFAULT = 512;
-
- /** Minimum virtual CPU cores used for allocating an opportunistic container.
- * */
- public static final String OPPORTUNISTIC_CONTAINERS_MIN_VCORES =
- YARN_PREFIX + "opportunistic-containers.min-vcores";
- public static final int OPPORTUNISTIC_CONTAINERS_MIN_VCORES_DEFAULT = 1;
-
- /** Maximum memory (in MB) used for allocating an opportunistic container. */
- public static final String OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB =
- YARN_PREFIX + "opportunistic-containers.max-memory-mb";
- public static final int OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB_DEFAULT = 2048;
-
- /** Maximum virtual CPU cores used for allocating an opportunistic container.
- * */
- public static final String OPPORTUNISTIC_CONTAINERS_MAX_VCORES =
- YARN_PREFIX + "opportunistic-containers.max-vcores";
- public static final int OPPORTUNISTIC_CONTAINERS_MAX_VCORES_DEFAULT = 4;
-
- /** Incremental memory (in MB) used for allocating an opportunistic container.
- * */
- public static final String OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB =
- YARN_PREFIX + "opportunistic-containers.incr-memory-mb";
- public static final int OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB_DEFAULT =
- 512;
-
- /** Incremental virtual CPU cores used for allocating an opportunistic
- * container. */
- public static final String OPPORTUNISTIC_CONTAINERS_INCR_VCORES =
- YARN_PREFIX + "opportunistic-containers.incr-vcores";
- public static final int OPPORTUNISTIC_CONTAINERS_INCR_VCORES_DEFAULT = 1;
-
- /** Container token expiry for opportunistic containers. */
- public static final String OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS =
- YARN_PREFIX + "opportunistic-containers.container-token-expiry-ms";
- public static final int OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS_DEFAULT =
- 600000;
-
/** Number of nodes to be used by the Opportunistic Container allocator for
* dispatching containers during container allocation. */
public static final String OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 47d12d1..c8c4edd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2766,63 +2766,6 @@
<property>
<description>
- Minimum memory (in MB) used for allocating an opportunistic container.
- </description>
- <name>yarn.opportunistic-containers.min-memory-mb</name>
- <value>512</value>
- </property>
-
- <property>
- <description>
- Minimum virtual CPU cores used for allocating an opportunistic container.
- </description>
- <name>yarn.opportunistic-containers.min-vcores</name>
- <value>1</value>
- </property>
-
- <property>
- <description>
- Maximum memory (in MB) used for allocating an opportunistic container.
- </description>
- <name>yarn.opportunistic-containers.max-memory-mb</name>
- <value>2048</value>
- </property>
-
- <property>
- <description>
- Maximum virtual CPU cores used for allocating an opportunistic container.
- </description>
- <name>yarn.opportunistic-containers.max-vcores</name>
- <value>4</value>
- </property>
-
- <property>
- <description>
- Incremental memory (in MB) used for allocating an opportunistic container.
- </description>
- <name>yarn.opportunistic-containers.incr-memory-mb</name>
- <value>512</value>
- </property>
-
- <property>
- <description>
- Incremental virtual CPU cores used for allocating an opportunistic
- container.
- </description>
- <name>yarn.opportunistic-containers.incr-vcores</name>
- <value>1</value>
- </property>
-
- <property>
- <description>
- Container token expiry for opportunistic containers.
- </description>
- <name>yarn.opportunistic-containers.container-token-expiry-ms</name>
- <value>600000</value>
- </property>
-
- <property>
- <description>
Number of nodes to be used by the Opportunistic Container Allocator for
dispatching containers during container allocation.
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 0c2b1ac..753bf04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -170,11 +170,11 @@ public class ContainerScheduler extends AbstractService implements
this.opportunisticContainersStatus.setWaitQueueLength(
getNumQueuedContainers());
this.opportunisticContainersStatus.setOpportMemoryUsed(
- metrics.getOpportMemoryUsed());
+ metrics.getAllocatedOpportunisticGB());
this.opportunisticContainersStatus.setOpportCoresUsed(
- metrics.getOpportCoresUsed());
+ metrics.getAllocatedOpportunisticVCores());
this.opportunisticContainersStatus.setRunningOpportContainers(
- metrics.getRunningOpportContainers());
+ metrics.getRunningOpportunisticContainers());
return this.opportunisticContainersStatus;
}
@@ -196,7 +196,7 @@ public class ContainerScheduler extends AbstractService implements
this.utilizationTracker.subtractContainerResource(container);
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
- this.metrics.opportunisticContainerCompleted(container);
+ this.metrics.completeOpportunisticContainer(container.getResource());
}
startPendingContainers();
}
@@ -298,7 +298,7 @@ public class ContainerScheduler extends AbstractService implements
this.utilizationTracker.addContainerResources(container);
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
- this.metrics.opportunisticContainerStarted(container);
+ this.metrics.startOpportunisticContainer(container.getResource());
}
container.sendLaunchEvent();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
index b001b63..291b488 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
@@ -29,8 +29,6 @@ import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.yarn.api.records.Resource;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container
- .Container;
@Metrics(about="Metrics for node manager", context="yarn")
public class NodeManagerMetrics {
@@ -64,12 +62,12 @@ public class NodeManagerMetrics {
@Metric("Disk utilization % on good log dirs")
MutableGaugeInt goodLogDirsDiskUtilizationPerc;
- @Metric("Memory used by Opportunistic Containers in MB")
- MutableGaugeLong opportMemoryUsed;
- @Metric("# of Virtual Cores used by opportunistic containers")
- MutableGaugeInt opportCoresUsed;
+ @Metric("Current allocated memory by opportunistic containers in GB")
+ MutableGaugeLong allocatedOpportunisticGB;
+ @Metric("Current allocated Virtual Cores by opportunistic containers")
+ MutableGaugeInt allocatedOpportunisticVCores;
@Metric("# of running opportunistic containers")
- MutableGaugeInt runningOpportContainers;
+ MutableGaugeInt runningOpportunisticContainers;
// CHECKSTYLE:ON:VisibilityModifier
@@ -77,6 +75,7 @@ public class NodeManagerMetrics {
private long allocatedMB;
private long availableMB;
+ private long allocatedOpportunisticMB;
public NodeManagerMetrics(JvmMetrics jvmMetrics) {
this.jvmMetrics = jvmMetrics;
@@ -141,30 +140,6 @@ public class NodeManagerMetrics {
containersReIniting.decr();
}
- public long getOpportMemoryUsed() {
- return opportMemoryUsed.value();
- }
-
- public int getOpportCoresUsed() {
- return opportCoresUsed.value();
- }
-
- public int getRunningOpportContainers() {
- return runningOpportContainers.value();
- }
-
- public void opportunisticContainerCompleted(Container container) {
- opportMemoryUsed.decr(container.getResource().getMemorySize());
- opportCoresUsed.decr(container.getResource().getVirtualCores());
- runningOpportContainers.decr();
- }
-
- public void opportunisticContainerStarted(Container container) {
- opportMemoryUsed.incr(container.getResource().getMemorySize());
- opportCoresUsed.incr(container.getResource().getVirtualCores());
- runningOpportContainers.incr();
- }
-
public void allocateContainer(Resource res) {
allocatedContainers.incr();
allocatedMB = allocatedMB + res.getMemorySize();
@@ -196,6 +171,22 @@ public class NodeManagerMetrics {
availableVCores.decr(deltaVCores);
}
+ public void startOpportunisticContainer(Resource res) {
+ runningOpportunisticContainers.incr();
+ allocatedOpportunisticMB = allocatedOpportunisticMB + res.getMemorySize();
+ allocatedOpportunisticGB
+ .set((int) Math.ceil(allocatedOpportunisticMB / 1024d));
+ allocatedOpportunisticVCores.incr(res.getVirtualCores());
+ }
+
+ public void completeOpportunisticContainer(Resource res) {
+ runningOpportunisticContainers.decr();
+ allocatedOpportunisticMB = allocatedOpportunisticMB - res.getMemorySize();
+ allocatedOpportunisticGB
+ .set((int) Math.ceil(allocatedOpportunisticMB / 1024d));
+ allocatedOpportunisticVCores.decr(res.getVirtualCores());
+ }
+
public void addResource(Resource res) {
availableMB = availableMB + res.getMemorySize();
availableGB.incr((int)Math.floor(availableMB/1024d));
@@ -272,4 +263,16 @@ public class NodeManagerMetrics {
public int getContainersRolledbackOnFailure() {
return containersRolledBackOnFailure.value();
}
+
+ public long getAllocatedOpportunisticGB() {
+ return allocatedOpportunisticGB.value();
+ }
+
+ public int getAllocatedOpportunisticVCores() {
+ return allocatedOpportunisticVCores.value();
+ }
+
+ public int getRunningOpportunisticContainers() {
+ return runningOpportunisticContainers.value();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
index 7814b84..7b7963f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
@@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRespons
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
@@ -199,11 +198,12 @@ public class OpportunisticContainerAllocatorAMService
}
});
int tokenExpiryInterval = getConfig()
- .getInt(YarnConfiguration.OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS,
- YarnConfiguration.
- OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS_DEFAULT);
- opCtx.updateAllocationParams(createMinContainerResource(),
- createMaxContainerResource(), createIncrContainerResource(),
+ .getInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS);
+ opCtx.updateAllocationParams(
+ rmContext.getScheduler().getMinimumResourceCapability(),
+ rmContext.getScheduler().getMaximumResourceCapability(),
+ rmContext.getScheduler().getMinimumResourceCapability(),
tokenExpiryInterval);
appAttempt.setOpportunisticContainerContext(opCtx);
}
@@ -273,14 +273,14 @@ public class OpportunisticContainerAllocatorAMService
RegisterDistributedSchedulingAMResponse dsResp = recordFactory
.newRecordInstance(RegisterDistributedSchedulingAMResponse.class);
dsResp.setRegisterResponse(response);
- dsResp.setMinContainerResource(createMinContainerResource());
- dsResp.setMaxContainerResource(createMaxContainerResource());
- dsResp.setIncrContainerResource(createIncrContainerResource());
+ dsResp.setMinContainerResource(
+ rmContext.getScheduler().getMinimumResourceCapability());
+ dsResp.setMaxContainerResource(
+ rmContext.getScheduler().getMaximumResourceCapability());
dsResp.setContainerTokenExpiryInterval(
getConfig().getInt(
- YarnConfiguration.OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS,
- YarnConfiguration.
- OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS_DEFAULT));
+ YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS));
dsResp.setContainerIdStart(
this.rmContext.getEpoch() << ResourceManager.EPOCH_BIT_SHIFT);
@@ -384,18 +384,6 @@ public class OpportunisticContainerAllocatorAMService
return nodeMonitor.getThresholdCalculator();
}
- private Resource createIncrContainerResource() {
- return Resource.newInstance(
- getConfig().getInt(
- YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB,
- YarnConfiguration.
- OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB_DEFAULT),
- getConfig().getInt(
- YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_VCORES,
- YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_VCORES_DEFAULT)
- );
- }
-
private synchronized List<RemoteNode> getLeastLoadedNodes() {
long currTime = System.currentTimeMillis();
if ((currTime - lastCacheUpdateTime > cacheRefreshInterval)
@@ -425,30 +413,6 @@ public class OpportunisticContainerAllocatorAMService
: null;
}
- private Resource createMaxContainerResource() {
- return Resource.newInstance(
- getConfig().getInt(
- YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB,
- YarnConfiguration
- .OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB_DEFAULT),
- getConfig().getInt(
- YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_VCORES,
- YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_VCORES_DEFAULT)
- );
- }
-
- private Resource createMinContainerResource() {
- return Resource.newInstance(
- getConfig().getInt(
- YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB,
- YarnConfiguration.
- OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB_DEFAULT),
- getConfig().getInt(
- YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_VCORES,
- YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_VCORES_DEFAULT)
- );
- }
-
private static ApplicationAttemptId getAppAttemptId() throws YarnException {
AMRMTokenIdentifier amrmTokenIdentifier =
YarnServerSecurityUtils.authorizeRequest();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 995a7b0..6744c2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -149,10 +149,10 @@ public abstract class SchedulerNode {
*/
public synchronized void allocateContainer(RMContainer rmContainer) {
Container container = rmContainer.getContainer();
- if (rmContainer.getExecutionType() != ExecutionType.OPPORTUNISTIC) {
+ if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
deductUnallocatedResource(container.getResource());
+ ++numContainers;
}
- ++numContainers;
launchedContainers.put(container.getId(), rmContainer);
@@ -251,8 +251,8 @@ public abstract class SchedulerNode {
Container container) {
if (container.getExecutionType() == ExecutionType.GUARANTEED) {
addUnallocatedResource(container.getResource());
+ --numContainers;
}
- --numContainers;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
index 7b68b84..c03df63 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
@@ -49,6 +49,7 @@ class NodesPage extends RmView {
static class NodesBlock extends HtmlBlock {
final ResourceManager rm;
private static final long BYTES_IN_MB = 1024 * 1024;
+ private static final long BYTES_IN_GB = 1024 * 1024 * 1024;
private static boolean opportunisticContainersEnabled;
@Inject
@@ -181,8 +182,9 @@ class NodesPage extends RmView {
nodeTableData
.append(String.valueOf(info.getNumRunningOpportContainers()))
.append("\",\"").append("<br title='")
- .append(String.valueOf(info.getUsedMemoryOpport())).append("'>")
- .append(StringUtils.byteDesc(info.getUsedMemoryOpport()))
+ .append(String.valueOf(info.getUsedMemoryOpportGB())).append("'>")
+ .append(StringUtils.byteDesc(
+ info.getUsedMemoryOpportGB() * BYTES_IN_GB))
.append("\",\"")
.append(String.valueOf(info.getUsedVirtualCoresOpport()))
.append("\",\"")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index aee7717..3416e52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -51,7 +51,7 @@ public class NodeInfo {
protected long usedVirtualCores;
protected long availableVirtualCores;
private int numRunningOpportContainers;
- private long usedMemoryOpport; // Memory in bytes.
+ private long usedMemoryOpportGB;
private long usedVirtualCoresOpport;
private int numQueuedContainers;
protected ArrayList<String> nodeLabels = new ArrayList<String>();
@@ -85,7 +85,7 @@ public class NodeInfo {
// Status of opportunistic containers.
this.numRunningOpportContainers = 0;
- this.usedMemoryOpport = 0;
+ this.usedMemoryOpportGB = 0;
this.usedVirtualCoresOpport = 0;
this.numQueuedContainers = 0;
OpportunisticContainersStatus opportStatus =
@@ -93,7 +93,7 @@ public class NodeInfo {
if (opportStatus != null) {
this.numRunningOpportContainers =
opportStatus.getRunningOpportContainers();
- this.usedMemoryOpport = opportStatus.getOpportMemoryUsed();
+ this.usedMemoryOpportGB = opportStatus.getOpportMemoryUsed();
this.usedVirtualCoresOpport = opportStatus.getOpportCoresUsed();
this.numQueuedContainers = opportStatus.getQueuedOpportContainers();
}
@@ -165,8 +165,8 @@ public class NodeInfo {
return numRunningOpportContainers;
}
- public long getUsedMemoryOpport() {
- return usedMemoryOpport;
+ public long getUsedMemoryOpportGB() {
+ return usedMemoryOpportGB;
}
public long getUsedVirtualCoresOpport() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index 10aa92a..fb597fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -726,7 +726,7 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
"aggregatedContainersVirtualMemoryMB"),
WebServicesTestUtils.getXmlFloat(element, "containersCPUUsage"),
WebServicesTestUtils.getXmlInt(element, "numRunningOpportContainers"),
- WebServicesTestUtils.getXmlLong(element, "usedMemoryOpport"),
+ WebServicesTestUtils.getXmlLong(element, "usedMemoryOpportGB"),
WebServicesTestUtils.getXmlInt(element, "usedVirtualCoresOpport"),
WebServicesTestUtils.getXmlInt(element, "numQueuedContainers"));
}
@@ -753,7 +753,7 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
resourceInfo.getInt("aggregatedContainersVirtualMemoryMB"),
resourceInfo.getDouble("containersCPUUsage"),
nodeInfo.getInt("numRunningOpportContainers"),
- nodeInfo.getLong("usedMemoryOpport"),
+ nodeInfo.getLong("usedMemoryOpportGB"),
nodeInfo.getInt("usedVirtualCoresOpport"),
nodeInfo.getInt("numQueuedContainers"));
}
@@ -766,7 +766,7 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
int nodePhysicalMemoryMB, int nodeVirtualMemoryMB, double nodeCPUUsage,
int containersPhysicalMemoryMB, int containersVirtualMemoryMB,
double containersCPUUsage, int numRunningOpportContainers,
- long usedMemoryOpport, int usedVirtualCoresOpport,
+ long usedMemoryOpportGB, int usedVirtualCoresOpport,
int numQueuedContainers)
throws JSONException, Exception {
@@ -827,8 +827,8 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
numRunningOpportContainers,
opportunisticStatus.getRunningOpportContainers(),
numRunningOpportContainers);
- assertEquals("usedMemoryOpport doesn't match: " + usedMemoryOpport,
- opportunisticStatus.getOpportMemoryUsed(), usedMemoryOpport);
+ assertEquals("usedMemoryOpportGB doesn't match: " + usedMemoryOpportGB,
+ opportunisticStatus.getOpportMemoryUsed(), usedMemoryOpportGB);
assertEquals(
"usedVirtualCoresOpport doesn't match: " + usedVirtualCoresOpport,
opportunisticStatus.getOpportCoresUsed(), usedVirtualCoresOpport);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org