You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/01/29 05:10:42 UTC
[24/50] [abbrv] hadoop git commit: YARN-7817. Add Resource reference
to RM's NodeInfo object so REST API can get non memory/vcore resource usages.
(Sunil G via wangda)
YARN-7817. Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore resource usages. (Sunil G via wangda)
Change-Id: Ia7ceeabd82046645ddeaf487c763288f36cfbdee
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0cfb0a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0cfb0a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0cfb0a3
Branch: refs/heads/YARN-1011
Commit: e0cfb0a31a1c6ad0f9dcf0705b44229593a0915f
Parents: 2e58656
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Jan 26 15:43:27 2018 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Jan 26 15:43:27 2018 +0800
----------------------------------------------------------------------
.../webapp/JAXBContextResolver.java | 2 +-
.../webapp/dao/ClusterMetricsInfo.java | 24 ++++++++++
.../resourcemanager/webapp/dao/NodeInfo.java | 20 ++++++++
.../webapp/dao/ResourceInfo.java | 18 ++++++--
.../webapp/dao/ResourceInformationsInfo.java | 48 ++++++++++++++++++++
.../main/webapp/app/models/cluster-metric.js | 4 +-
.../src/main/webapp/app/models/yarn-rm-node.js | 4 +-
.../main/webapp/app/serializers/yarn-rm-node.js | 4 +-
8 files changed, 114 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
index 2f50a24..2e4204e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
@@ -55,7 +55,7 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> {
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
- AppTimeoutInfo.class };
+ AppTimeoutInfo.class, ResourceInformationsInfo.class };
// these dao classes need root unwrapping
final Class[] rootUnwrappedTypes =
{ NewApplication.class, ApplicationSubmissionContextInfo.class,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index 3214cb9..84f70d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -61,6 +61,12 @@ public class ClusterMetricsInfo {
private int activeNodes;
private int shutdownNodes;
+ // Total used resource of the cluster, including all partitions
+ private ResourceInfo totalUsedResourcesAcrossPartition;
+
+ // Total registered resources of the cluster, including all partitions
+ private ResourceInfo totalClusterResourcesAcrossPartition;
+
public ClusterMetricsInfo() {
} // JAXB needs this
@@ -92,9 +98,20 @@ public class ClusterMetricsInfo {
this.containersReserved = metrics.getReservedContainers();
if (rs instanceof CapacityScheduler) {
+ CapacityScheduler cs = (CapacityScheduler) rs;
this.totalMB = availableMB + allocatedMB + reservedMB;
this.totalVirtualCores =
availableVirtualCores + allocatedVirtualCores + containersReserved;
+ // TODO, add support of other schedulers to get total used resources
+ // across partition.
+ if (cs.getRootQueue() != null
+ && cs.getRootQueue().getQueueResourceUsage() != null
+ && cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) {
+ totalUsedResourcesAcrossPartition = new ResourceInfo(
+ cs.getRootQueue().getQueueResourceUsage().getAllUsed());
+ totalClusterResourcesAcrossPartition = new ResourceInfo(
+ cs.getClusterResource());
+ }
} else {
this.totalMB = availableMB + allocatedMB;
this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
@@ -310,4 +327,11 @@ public class ClusterMetricsInfo {
this.shutdownNodes = shutdownNodes;
}
+ public ResourceInfo getTotalUsedResourcesAcrossPartition() {
+ return totalUsedResourcesAcrossPartition;
+ }
+
+ public ResourceInfo getTotalClusterResourcesAcrossPartition() {
+ return totalClusterResourcesAcrossPartition;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index 2530c8e..3cec215 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -58,6 +58,8 @@ public class NodeInfo {
private int numQueuedContainers;
protected ArrayList<String> nodeLabels = new ArrayList<String>();
protected ResourceUtilizationInfo resourceUtilization;
+ protected ResourceInfo usedResource;
+ protected ResourceInfo availableResource;
public NodeInfo() {
} // JAXB needs this
@@ -75,6 +77,8 @@ public class NodeInfo {
this.usedVirtualCores = report.getUsedResource().getVirtualCores();
this.availableVirtualCores =
report.getAvailableResource().getVirtualCores();
+ this.usedResource = new ResourceInfo(report.getUsedResource());
+ this.availableResource = new ResourceInfo(report.getAvailableResource());
}
this.id = id.toString();
this.rack = ni.getRackName();
@@ -183,6 +187,22 @@ public class NodeInfo {
return this.nodeLabels;
}
+ public ResourceInfo getUsedResource() {
+ return usedResource;
+ }
+
+ public void setUsedResource(ResourceInfo used) {
+ this.usedResource = used;
+ }
+
+ public ResourceInfo getAvailableResource() {
+ return availableResource;
+ }
+
+ public void setAvailableResource(ResourceInfo avail) {
+ this.availableResource = avail;
+ }
+
public ResourceUtilizationInfo getResourceUtilization() {
return this.resourceUtilization;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
index e13980a..5bed936 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
@@ -26,6 +26,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.resource.Resources;
+
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
public class ResourceInfo {
@@ -34,6 +35,9 @@ public class ResourceInfo {
long memory;
@XmlElement
int vCores;
+ @XmlElement
+ ResourceInformationsInfo resourceInformations =
+ new ResourceInformationsInfo();
private Resource resources;
@@ -41,9 +45,13 @@ public class ResourceInfo {
}
public ResourceInfo(Resource res) {
- memory = res.getMemorySize();
- vCores = res.getVirtualCores();
- resources = Resources.clone(res);
+ // Make sure no NPE.
+ if (res != null) {
+ memory = res.getMemorySize();
+ vCores = res.getVirtualCores();
+ resources = Resources.clone(res);
+ resourceInformations.addAll(res.getAllResourcesListCopy());
+ }
}
public long getMemorySize() {
@@ -84,4 +92,8 @@ public class ResourceInfo {
public Resource getResource() {
return Resource.newInstance(resources);
}
+
+ public ResourceInformationsInfo getResourcesInformations() {
+ return resourceInformations;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java
new file mode 100644
index 0000000..651d7df
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+
+@XmlRootElement(name = "resourceInformations")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ResourceInformationsInfo {
+
+ @XmlElement(name = "resourceInformation")
+ protected ArrayList<ResourceInformation> resourceInformation =
+ new ArrayList<ResourceInformation>();
+
+ public ResourceInformationsInfo() {
+ } // JAXB needs this
+
+ public ArrayList<ResourceInformation> getApps() {
+ return resourceInformation;
+ }
+
+ public void addAll(List<ResourceInformation> resourcesInformationsInfo) {
+ resourceInformation.addAll(resourcesInformationsInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
index bbc6f08..764723a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
@@ -169,8 +169,8 @@ export default DS.Model.extend({
&& this.get("totalUsedResourcesAcrossPartition")) {
var usages = [];
- var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourcesInformations;
- var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourcesInformations;
+ var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourceInformations.resourceInformation;
+ var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourceInformations.resourceInformation;
clusterResourceInformations.forEach(function(cluster) {
var perResourceTypeUsage = {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
index 3b2385f..64fe7b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
@@ -99,7 +99,7 @@ export default DS.Model.extend({
const usedResource = this.get("usedResource");
const availableResource = this.get("availableResource");
- var resourceInformations = usedResource ? usedResource.resourcesInformations : [];
+ var resourceInformations = usedResource ? usedResource.resourceInformations.resourceInformation : [];
for (var i = 0; i < resourceInformations.length; i++) {
ri = resourceInformations[i];
if (ri.name === "yarn.io/gpu") {
@@ -108,7 +108,7 @@ export default DS.Model.extend({
}
var available = 0;
- resourceInformations = availableResource ? availableResource.resourcesInformations : [];
+ resourceInformations = availableResource ? availableResource.resourceInformations.resourceInformation : [];
for (i = 0; i < resourceInformations.length; i++) {
ri = resourceInformations[i];
if (ri.name === "yarn.io/gpu") {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
index a3a1d59..c27fd92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
@@ -42,8 +42,8 @@ export default DS.JSONAPISerializer.extend({
availableVirtualCores: payload.availableVirtualCores,
version: payload.version,
nodeLabels: payload.nodeLabels,
- usedResource: payload.used,
- availableResource: payload.avail
+ usedResource: payload.usedResource,
+ availableResource: payload.availableResource
}
};
return fixedPayload;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org