You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2019/01/04 22:28:48 UTC
[hadoop] branch YARN-8200 updated (14dc25e -> f76f2cf)
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a change to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 14dc25e YARN-7033. Add support for NM Recovery of assigned resources (e.g. GPU's, NUMA, FPGA's) to container. (Devaraj K and Wangda Tan)
new e9ce716 Partial commit of YARN-7224 to get GpuDevice class and changes
new 1f2e67b YARN-7330. Add support to show GPU in UI including metrics. Contributed by Wangda Tan.
new e4268da YARN-7345. GPU Isolation: Incorrect minor device numbers written to devices.deny file. (Jonathan Hung via wangda)
new b69e841 YARN-7573. Gpu Information page could be empty for nodes without GPU. (Sunil G via wangda)
new c9d0000 YARN-7637. GPU volume creation command fails when work preserving is disabled at NM. Contributed by Zian Chen.
new c3e4d9f YARN-7594. TestNMWebServices#testGetNMResourceInfo fails on trunk. Contributed by Gergely Novák.
new 0330369 YARN-9175. Null resources check in ResourceInfo for branch-3.0
new e4ede15 YARN-7143. FileNotFound handling in ResourceUtils is inconsistent
new f76f2cf YARN-7383. Node resource is not parsed correctly for resource names containing dot. Contributed by Gergely Novák.
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hadoop-yarn/dev-support/findbugs-exclude.xml | 8 +
.../apache/hadoop/yarn/api/records/Resource.java | 20 +++
.../hadoop/yarn/util/resource/ResourceUtils.java | 61 +++----
.../yarn/util/resource/TestResourceUtils.java | 5 +-
.../resources/resource-types/node-resources-2.xml | 5 +
.../resources/resource-types/resource-types-4.xml | 7 +-
.../nodemanager/DockerContainerExecutor.java | 2 +-
.../container/ResourceMappings.java | 2 +-
.../linux/resources/gpu/GpuResourceAllocator.java | 119 +++++++------
.../resources/gpu/GpuResourceHandlerImpl.java | 27 +--
.../resourceplugin/ResourcePlugin.java | 11 ++
.../resourceplugin/gpu/AssignedGpuDevice.java | 83 +++++++++
.../resourceplugin/gpu/GpuDevice.java | 82 +++++++++
.../resourceplugin/gpu/GpuDiscoverer.java | 30 ++--
.../gpu/GpuNodeResourceUpdateHandler.java | 10 +-
.../resourceplugin/gpu/GpuResourcePlugin.java | 24 ++-
.../recovery/NMLeveldbStateStoreService.java | 68 ++++----
.../recovery/NMNullStateStoreService.java | 6 +-
.../nodemanager/recovery/NMStateStoreService.java | 17 +-
.../server/nodemanager/webapp/NMWebServices.java | 25 +++
.../nodemanager/webapp/dao/NMResourceInfo.java} | 14 +-
.../webapp/dao/gpu/GpuDeviceInformation.java | 2 +-
.../webapp/dao/gpu/NMGpuResourceInfo.java | 80 +++++++++
.../webapp/dao/gpu/PerGpuDeviceInformation.java | 2 +-
.../webapp/dao/gpu/PerGpuMemoryUsage.java | 2 +-
.../impl/modules/gpu/gpu-module.c | 2 +-
.../test/modules/gpu/test-gpu-module.cc | 13 ++
.../TestDockerContainerExecutorWithMocks.java | 2 +-
.../TestContainerManagerRecovery.java | 15 +-
.../resources/gpu/TestGpuResourceHandler.java | 194 +++++++++++++++------
.../resourceplugin/gpu/TestGpuDiscoverer.java | 34 +++-
.../recovery/NMMemoryStateStoreService.java | 10 +-
.../recovery/TestNMLeveldbStateStoreService.java | 28 ++-
.../nodemanager/webapp/TestNMWebServices.java | 175 +++++++++++++++----
.../dao/gpu/TestGpuDeviceInformationParser.java | 2 +-
.../resourcemanager/webapp/dao/ResourceInfo.java | 5 +-
.../app/adapters/{yarn-node.js => yarn-nm-gpu.js} | 4 +-
.../src/main/webapp/app/components/donut-chart.js | 20 ++-
.../main/webapp/app/components/gpu-donut-chart.js | 66 +++++++
.../src/main/webapp/app/constants.js | 13 ++
.../webapp/app/controllers/yarn-nodes/table.js | 2 +-
.../src/main/webapp/app/models/cluster-metric.js | 69 ++++++++
.../app/models/{yarn-entity.js => yarn-nm-gpu.js} | 9 +-
.../webapp/app/models/yarn-queue/capacity-queue.js | 3 +-
.../src/main/webapp/app/models/yarn-rm-node.js | 37 ++++
.../hadoop-yarn-ui/src/main/webapp/app/router.js | 5 +-
.../src/main/webapp/app/routes/cluster-overview.js | 2 +-
.../src/main/webapp/app/routes/yarn-node.js | 2 +
.../table.js => yarn-node/yarn-nm-gpu.js} | 0
.../{yarn-container-log.js => yarn-nm-gpu.js} | 25 +--
.../app/serializers/yarn-queue/capacity-queue.js | 1 +
.../main/webapp/app/serializers/yarn-rm-node.js | 4 +-
.../main/webapp/app/templates/cluster-overview.hbs | 88 +++++++---
.../app/templates/components/node-menu-panel.hbs | 10 +-
.../app/templates/components/yarn-nm-gpu-info.hbs | 69 ++++++++
.../main/webapp/app/templates/yarn-node-apps.hbs | 2 +-
.../webapp/app/templates/yarn-node-containers.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-node.hbs | 125 -------------
.../main/webapp/app/templates/yarn-node/info.hbs | 154 ++++++++++++++++
.../yarn-nm-gpu.hbs} | 52 +++---
.../src/main/webapp/app/utils/converter.js | 51 ++++++
61 files changed, 1516 insertions(+), 491 deletions(-)
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java
copy hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/{hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDeleteResponseInfo.java => hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java} (77%)
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java
copy hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/{yarn-node.js => yarn-nm-gpu.js} (93%)
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js
copy hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/{yarn-entity.js => yarn-nm-gpu.js} (88%)
copy hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/{yarn-nodes/table.js => yarn-node/yarn-nm-gpu.js} (100%)
copy hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/{yarn-container-log.js => yarn-nm-gpu.js} (73%)
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-nm-gpu-info.hbs
delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node.hbs
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/info.hbs
copy hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/{yarn-node-apps.hbs => yarn-node/yarn-nm-gpu.hbs} (50%)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 05/09: YARN-7637. GPU volume creation command fails when
work preserving is disabled at NM. Contributed by Zian Chen.
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit c9d000057c1373e05a22a956e82635395692ea98
Author: Sunil G <su...@apache.org>
AuthorDate: Thu Mar 1 22:39:32 2018 +0530
YARN-7637. GPU volume creation command fails when work preserving is disabled at NM. Contributed by Zian Chen.
---
.../recovery/NMNullStateStoreService.java | 1 +
.../resources/gpu/TestGpuResourceHandler.java | 30 ++++++++++++++++++++++
2 files changed, 31 insertions(+)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
index 77ed241..80a1d05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
@@ -272,6 +272,7 @@ public class NMNullStateStoreService extends NMStateStoreService {
public void storeAssignedResources(Container container,
String resourceType, List<? extends Serializable> assignedResources)
throws IOException {
+ updateContainerResourceMapping(container, resourceType, assignedResources);
}
@Override
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
index 4e89e65..9799686 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDiscoverer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
import org.junit.Assert;
@@ -343,6 +344,35 @@ public class TestGpuResourceHandler {
}
@Test
+ public void testAllocationStoredWithNULLStateStore() throws Exception {
+ NMNullStateStoreService mockNMNULLStateStore = mock(NMNullStateStoreService.class);
+
+ Context nmnctx = mock(Context.class);
+ when(nmnctx.getNMStateStore()).thenReturn(mockNMNULLStateStore);
+
+ GpuResourceHandlerImpl gpuNULLStateResourceHandler =
+ new GpuResourceHandlerImpl(nmnctx, mockCGroupsHandler,
+ mockPrivilegedExecutor);
+
+ Configuration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
+ GpuDiscoverer.getInstance().initialize(conf);
+
+ gpuNULLStateResourceHandler.bootstrap(conf);
+ Assert.assertEquals(4,
+ gpuNULLStateResourceHandler.getGpuAllocator().getAvailableGpus());
+
+ /* Start container 1, asks 3 containers */
+ Container container = mockContainerWithGpuRequest(1, 3);
+ gpuNULLStateResourceHandler.preStart(container);
+
+ verify(nmnctx.getNMStateStore()).storeAssignedResources(container,
+ ResourceInformation.GPU_URI, Arrays
+ .asList(new GpuDevice(0, 0), new GpuDevice(1, 1),
+ new GpuDevice(2, 3)));
+ }
+
+ @Test
public void testRecoverResourceAllocation() throws Exception {
Configuration conf = new YarnConfiguration();
conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/09: YARN-7330. Add support to show GPU in UI including
metrics. Contributed by Wangda Tan.
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 1f2e67b0caaff159ba6fd275a12aa2610cde176b
Author: Sunil G <su...@apache.org>
AuthorDate: Thu Nov 23 07:54:20 2017 +0530
YARN-7330. Add support to show GPU in UI including metrics. Contributed by Wangda Tan.
---
.../hadoop-yarn/dev-support/findbugs-exclude.xml | 8 +
.../apache/hadoop/yarn/api/records/Resource.java | 20 +++
.../linux/resources/gpu/GpuResourceAllocator.java | 19 ++-
.../resources/gpu/GpuResourceHandlerImpl.java | 1 -
.../resourceplugin/ResourcePlugin.java | 11 ++
.../gpu/{GpuDevice.java => AssignedGpuDevice.java} | 57 ++++---
.../resourceplugin/gpu/GpuDevice.java | 8 +-
.../resourceplugin/gpu/GpuResourcePlugin.java | 24 ++-
.../server/nodemanager/webapp/NMWebServices.java | 27 ++++
.../nodemanager/webapp/dao/NMResourceInfo.java} | 16 +-
.../webapp/dao/gpu/GpuDeviceInformation.java | 2 +-
.../webapp/dao/gpu/NMGpuResourceInfo.java | 80 +++++++++
.../webapp/dao/gpu/PerGpuDeviceInformation.java | 2 +-
.../webapp/dao/gpu/PerGpuMemoryUsage.java | 2 +-
.../resources/gpu/TestGpuResourceHandler.java | 6 +-
.../nodemanager/webapp/TestNMWebServices.java | 179 +++++++++++++++++----
.../dao/gpu/TestGpuDeviceInformationParser.java | 2 +-
.../app/{constants.js => adapters/yarn-nm-gpu.js} | 21 ++-
.../src/main/webapp/app/components/donut-chart.js | 20 ++-
.../main/webapp/app/components/gpu-donut-chart.js | 66 ++++++++
.../src/main/webapp/app/constants.js | 13 ++
.../webapp/app/controllers/yarn-nodes/table.js | 2 +-
.../src/main/webapp/app/models/cluster-metric.js | 69 ++++++++
.../app/{constants.js => models/yarn-nm-gpu.js} | 15 +-
.../webapp/app/models/yarn-queue/capacity-queue.js | 3 +-
.../src/main/webapp/app/models/yarn-rm-node.js | 35 ++++
.../hadoop-yarn-ui/src/main/webapp/app/router.js | 5 +-
.../src/main/webapp/app/routes/cluster-overview.js | 2 +-
.../src/main/webapp/app/routes/yarn-node.js | 2 +
.../yarn-node/yarn-nm-gpu.js} | 10 +-
.../yarn-node.js => serializers/yarn-nm-gpu.js} | 34 ++--
.../app/serializers/yarn-queue/capacity-queue.js | 1 +
.../main/webapp/app/serializers/yarn-rm-node.js | 4 +-
.../main/webapp/app/templates/cluster-overview.hbs | 88 ++++++----
.../app/templates/components/node-menu-panel.hbs | 10 +-
.../app/templates/components/yarn-nm-gpu-info.hbs | 69 ++++++++
.../src/main/webapp/app/templates/yarn-node.hbs | 125 --------------
.../main/webapp/app/templates/yarn-node/info.hbs | 154 ++++++++++++++++++
.../webapp/app/templates/yarn-node/yarn-nm-gpu.hbs | 53 ++++++
.../src/main/webapp/app/utils/converter.js | 51 ++++++
40 files changed, 1043 insertions(+), 273 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 45aa868..e6dcefb 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -633,4 +633,12 @@
<Method name="getResources" />
<Bug pattern="EI_EXPOSE_REP" />
</Match>
+
+ <!-- EQ_OVERRIDING_EQUALS_NOT_SYMMETRIC -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice" />
+ <Method name="equals" />
+ <Bug pattern="EQ_OVERRIDING_EQUALS_NOT_SYMMETRIC" />
+ </Match>
+
</FindBugsFilter>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 9a5bc79..122296e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -18,8 +18,12 @@
package org.apache.hadoop.yarn.api.records;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -213,6 +217,22 @@ public abstract class Resource implements Comparable<Resource> {
}
/**
+ * Get list of resource information, this will be used by JAXB.
+ * @return list of resources copy.
+ */
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ public List<ResourceInformation> getAllResourcesListCopy() {
+ List<ResourceInformation> list = new ArrayList<>();
+ for (ResourceInformation i : resources) {
+ ResourceInformation ri = new ResourceInformation();
+ ResourceInformation.copy(i, ri);
+ list.add(ri);
+ }
+ return list;
+ }
+
+ /**
* Get ResourceInformation for a specified resource.
*
* @param resource name of the resource
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
index f2bb342..5bdffc3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
@@ -30,11 +30,13 @@ import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -224,7 +226,20 @@ public class GpuResourceAllocator {
}
@VisibleForTesting
- public synchronized Map<GpuDevice, ContainerId> getDeviceAllocationMapping() {
- return new HashMap<>(usedDevices);
+ public synchronized Map<GpuDevice, ContainerId> getDeviceAllocationMappingCopy() {
+ return new HashMap<>(usedDevices);
+ }
+
+ public synchronized List<GpuDevice> getAllowedGpusCopy() {
+ return new ArrayList<>(allowedGpuDevices);
+ }
+
+ public synchronized List<AssignedGpuDevice> getAssignedGpusCopy() {
+ List<AssignedGpuDevice> assigns = new ArrayList<>();
+ for (Map.Entry<GpuDevice, ContainerId> entry : usedDevices.entrySet()) {
+ assigns.add(new AssignedGpuDevice(entry.getKey().getIndex(),
+ entry.getKey().getMinorNumber(), entry.getValue()));
+ }
+ return assigns;
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
index 4a783d3..5003821 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
@@ -133,7 +133,6 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
return ret;
}
- @VisibleForTesting
public GpuResourceAllocator getGpuAllocator() {
return gpuAllocator;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java
index 6e134b3..78167c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
/**
* {@link ResourcePlugin} is an interface for node manager to easier support
@@ -80,4 +81,14 @@ public interface ResourcePlugin {
* @throws YarnException if any issue occurs
*/
void cleanup() throws YarnException;
+
+ /**
+ * Get resource information from this plugin.
+ *
+ * @return NMResourceInfo, an example is
+ * {@link org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation}
+ *
+ * @throws YarnException when any issue occurs
+ */
+ NMResourceInfo getNMResourceInfo() throws YarnException;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java
similarity index 52%
copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java
copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java
index 8119924..d5d6f1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java
@@ -15,64 +15,69 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu;
-import java.io.Serializable;
+import org.apache.hadoop.yarn.api.records.ContainerId;
/**
- * This class is used to represent GPU device while allocation.
+ * In addition to {@link GpuDevice}, this include container id and more runtime
+ * information related to who is using the GPU device if possible
*/
-public class GpuDevice implements Serializable, Comparable {
- private int index;
- private int minorNumber;
- private static final long serialVersionUID = -6812314470754667710L;
+public class AssignedGpuDevice extends GpuDevice {
+ private static final long serialVersionUID = -12983712986315L;
+
+ String containerId;
+
+ public AssignedGpuDevice() {
- public GpuDevice(int index, int minorNumber) {
- this.index = index;
- this.minorNumber = minorNumber;
}
- public int getIndex() {
- return index;
+ public AssignedGpuDevice(int index, int minorNumber,
+ ContainerId containerId) {
+ super(index, minorNumber);
+ this.containerId = containerId.toString();
}
- public int getMinorNumber() {
- return minorNumber;
+ public String getContainerId() {
+ return containerId;
+ }
+
+ public void setContainerId(String containerId) {
+ this.containerId = containerId;
}
@Override
public boolean equals(Object obj) {
- if (obj == null || !(obj instanceof GpuDevice)) {
+ if (obj == null || !(obj instanceof AssignedGpuDevice)) {
return false;
}
- GpuDevice other = (GpuDevice) obj;
- return index == other.index && minorNumber == other.minorNumber;
+ AssignedGpuDevice other = (AssignedGpuDevice) obj;
+ return index == other.index && minorNumber == other.minorNumber
+ && containerId.equals(other.containerId);
}
@Override
public int compareTo(Object obj) {
- if (obj == null || (!(obj instanceof GpuDevice))) {
+ if (obj == null || (!(obj instanceof AssignedGpuDevice))) {
return -1;
}
- GpuDevice other = (GpuDevice) obj;
+ AssignedGpuDevice other = (AssignedGpuDevice) obj;
int result = Integer.compare(index, other.index);
if (0 != result) {
return result;
}
- return Integer.compare(minorNumber, other.minorNumber);
+ result = Integer.compare(minorNumber, other.minorNumber);
+ if (0 != result) {
+ return result;
+ }
+ return containerId.compareTo(other.containerId);
}
@Override
public int hashCode() {
final int prime = 47;
- return prime * index + minorNumber;
- }
-
- @Override
- public String toString() {
- return "(index=" + index + ",minor_number=" + minorNumber + ")";
+ return prime * (prime * index + minorNumber) + containerId.hashCode();
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java
index 8119924..082dc8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java
@@ -24,10 +24,14 @@ import java.io.Serializable;
* This class is used to represent GPU device while allocation.
*/
public class GpuDevice implements Serializable, Comparable {
- private int index;
- private int minorNumber;
+ protected int index;
+ protected int minorNumber;
private static final long serialVersionUID = -6812314470754667710L;
+ public GpuDevice() {
+
+ }
+
public GpuDevice(int index, int minorNumber) {
this.index = index;
this.minorNumber = minorNumber;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java
index 9576ce7..d294503 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java
@@ -18,17 +18,25 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu.GpuResourceAllocator;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu.GpuResourceHandlerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.NMGpuResourceInfo;
+
+import java.util.List;
+import java.util.Map;
public class GpuResourcePlugin implements ResourcePlugin {
- private ResourceHandler gpuResourceHandler = null;
+ private GpuResourceHandlerImpl gpuResourceHandler = null;
private GpuNodeResourceUpdateHandler resourceDiscoverHandler = null;
@Override
@@ -58,4 +66,18 @@ public class GpuResourcePlugin implements ResourcePlugin {
public void cleanup() throws YarnException {
// Do nothing.
}
+
+ @Override
+ public NMResourceInfo getNMResourceInfo() throws YarnException {
+ GpuDeviceInformation gpuDeviceInformation =
+ GpuDiscoverer.getInstance().getGpuDeviceInformation();
+ GpuResourceAllocator gpuResourceAllocator =
+ gpuResourceHandler.getGpuAllocator();
+ List<GpuDevice> totalGpus = gpuResourceAllocator.getAllowedGpusCopy();
+ List<AssignedGpuDevice> assignedGpuDevices =
+ gpuResourceAllocator.getAssignedGpusCopy();
+
+ return new NMGpuResourceInfo(gpuDeviceInformation, totalGpus,
+ assignedGpuDevices);
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index 60905d7..7476d75 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -27,6 +27,10 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
+
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -488,6 +492,29 @@ public class NMWebServices {
}
}
+ @GET
+ @Path("/resources/{resourcename}")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Object getNMResourceInfo(
+ @PathParam("resourcename")
+ String resourceName) throws YarnException {
+ init();
+ ResourcePluginManager rpm = this.nmContext.getResourcePluginManager();
+ if (rpm != null && rpm.getNameToPlugins() != null) {
+ ResourcePlugin plugin = rpm.getNameToPlugins().get(resourceName);
+ if (plugin != null) {
+ NMResourceInfo nmResourceInfo = plugin.getNMResourceInfo();
+ if (nmResourceInfo != null) {
+ return nmResourceInfo;
+ }
+ }
+ }
+
+ throw new YarnException(
+ "Could not get detailed resource information for given resource-name="
+ + resourceName);
+ }
+
private long parseLongParam(String bytes) {
if (bytes == null || bytes.isEmpty()) {
return Long.MAX_VALUE;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java
similarity index 73%
copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java
index d2937a0..18ce8ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java
@@ -16,9 +16,13 @@
* limitations under the License.
*/
-/**
- * Application level global constants go here.
- */
-export default {
- PARAM_SEPARATOR: '!',
-};
+package org.apache.hadoop.yarn.server.nodemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class NMResourceInfo {
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java
index 977032a..837d5cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java
@@ -25,7 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import java.util.List;
/**
- * All GPU Device Information in the system.
+ * All GPU Device Information in the system, fetched from nvidia-smi.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java
new file mode 100644
index 0000000..e9e00ed
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
+
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.List;
+
+/**
+ * Gpu device information return to client when
+ * {@link org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices#getNMResourceInfo(String)}
+ * is invoked.
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class NMGpuResourceInfo extends NMResourceInfo {
+ GpuDeviceInformation gpuDeviceInformation;
+
+ List<GpuDevice> totalGpuDevices;
+ List<AssignedGpuDevice> assignedGpuDevices;
+
+ public NMGpuResourceInfo() {
+
+ }
+
+ public NMGpuResourceInfo(GpuDeviceInformation gpuDeviceInformation,
+ List<GpuDevice> totalGpuDevices,
+ List<AssignedGpuDevice> assignedGpuDevices) {
+ this.gpuDeviceInformation = gpuDeviceInformation;
+ this.totalGpuDevices = totalGpuDevices;
+ this.assignedGpuDevices = assignedGpuDevices;
+ }
+
+ public GpuDeviceInformation getGpuDeviceInformation() {
+ return gpuDeviceInformation;
+ }
+
+ public void setGpuDeviceInformation(
+ GpuDeviceInformation gpuDeviceInformation) {
+ this.gpuDeviceInformation = gpuDeviceInformation;
+ }
+
+ public List<GpuDevice> getTotalGpuDevices() {
+ return totalGpuDevices;
+ }
+
+ public void setTotalGpuDevices(List<GpuDevice> totalGpuDevices) {
+ this.totalGpuDevices = totalGpuDevices;
+ }
+
+ public List<AssignedGpuDevice> getAssignedGpuDevices() {
+ return assignedGpuDevices;
+ }
+
+ public void setAssignedGpuDevices(
+ List<AssignedGpuDevice> assignedGpuDevices) {
+ this.assignedGpuDevices = assignedGpuDevices;
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java
index f315313..25c2e3a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java
@@ -135,7 +135,7 @@ public class PerGpuDeviceInformation {
this.gpuUtilizations = utilizations;
}
- @XmlElement(name = "bar1_memory_usage")
+ @XmlElement(name = "fb_memory_usage")
public PerGpuMemoryUsage getGpuMemoryUsage() {
return gpuMemoryUsage;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java
index 3964c4e..afc1a96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java
@@ -27,7 +27,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
@InterfaceAudience.Private
@InterfaceStability.Unstable
-@XmlRootElement(name = "bar1_memory_usage")
+@XmlRootElement(name = "fb_memory_usage")
public class PerGpuMemoryUsage {
long usedMemoryMiB = -1L;
long availMemoryMiB = -1L;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
index a4ac4bc..4e89e65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
@@ -368,7 +368,7 @@ public class TestGpuResourceHandler {
gpuResourceHandler.reacquireContainer(getContainerId(1));
Map<GpuDevice, ContainerId> deviceAllocationMapping =
- gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
+ gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy();
Assert.assertEquals(2, deviceAllocationMapping.size());
Assert.assertTrue(
deviceAllocationMapping.keySet().contains(new GpuDevice(1, 1)));
@@ -402,7 +402,7 @@ public class TestGpuResourceHandler {
// Make sure internal state not changed.
deviceAllocationMapping =
- gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
+ gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy();
Assert.assertEquals(2, deviceAllocationMapping.size());
Assert.assertTrue(deviceAllocationMapping.keySet()
.containsAll(Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 3))));
@@ -434,7 +434,7 @@ public class TestGpuResourceHandler {
// Make sure internal state not changed.
deviceAllocationMapping =
- gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
+ gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy();
Assert.assertEquals(2, deviceAllocationMapping.size());
Assert.assertTrue(deviceAllocationMapping.keySet()
.containsAll(Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 3))));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
index 4586a7b..c9c86b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
@@ -18,25 +18,17 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringReader;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.net.URL;
-import java.util.List;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.core.MediaType;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-
-import org.junit.Assert;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
@@ -48,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType;
import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo;
import org.apache.hadoop.yarn.logaggregation.TestContainerLogsUtils;
@@ -59,7 +52,15 @@ import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.NMGpuResourceInfo;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.PerGpuDeviceInformation;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams;
@@ -80,24 +81,32 @@ import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.servlet.GuiceServletContextListener;
-import com.google.inject.servlet.ServletModule;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.ClientResponse.Status;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.WebAppDescriptor;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringReader;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
/**
* Test the nodemanager node info web services api's
*/
public class TestNMWebServices extends JerseyTestBase {
- private static Context nmContext;
+ private static NodeManager.NMContext nmContext;
private static ResourceView resourceView;
private static ApplicationACLsManager aclsManager;
private static LocalDirsHandlerService dirsHandler;
@@ -411,6 +420,112 @@ public class TestNMWebServices extends JerseyTestBase {
assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID));
}
+ @Test
+ public void testGetNMResourceInfo()
+ throws YarnException, InterruptedException, JSONException {
+ ResourcePluginManager rpm = mock(ResourcePluginManager.class);
+ Map<String, ResourcePlugin> namesToPlugins = new HashMap<>();
+ ResourcePlugin mockPlugin1 = mock(ResourcePlugin.class);
+ NMResourceInfo nmResourceInfo1 = new NMResourceInfo() {
+ public long a = 1000L;
+ };
+ when(mockPlugin1.getNMResourceInfo()).thenReturn(nmResourceInfo1);
+ namesToPlugins.put("resource-1", mockPlugin1);
+ namesToPlugins.put("yarn.io/resource-1", mockPlugin1);
+ ResourcePlugin mockPlugin2 = mock(ResourcePlugin.class);
+ namesToPlugins.put("resource-2", mockPlugin2);
+ when(rpm.getNameToPlugins()).thenReturn(namesToPlugins);
+
+ nmContext.setResourcePluginManager(rpm);
+
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("node").path(
+ "resources").path("resource-2").accept(MediaType.APPLICATION_JSON).get(
+ ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON, response.getType().toString());
+
+ // Access resource-2 should fail (null NMResourceInfo returned).
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertIncludesException(json);
+
+ // Access resource-3 should fail (unkown plugin)
+ response = r.path("ws").path("v1").path("node").path(
+ "resources").path("resource-3").accept(MediaType.APPLICATION_JSON).get(
+ ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON, response.getType().toString());
+ json = response.getEntity(JSONObject.class);
+ assertIncludesException(json);
+
+ // Access resource-1 should success
+ response = r.path("ws").path("v1").path("node").path(
+ "resources").path("resource-1").accept(MediaType.APPLICATION_JSON).get(
+ ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON, response.getType().toString());
+ json = response.getEntity(JSONObject.class);
+ assertEquals(1000, json.get("a"));
+
+ // Access resource-1 should success (encoded yarn.io/Fresource-1).
+ response = r.path("ws").path("v1").path("node").path("resources").path(
+ "yarn.io%2Fresource-1").accept(MediaType.APPLICATION_JSON).get(
+ ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON, response.getType().toString());
+ json = response.getEntity(JSONObject.class);
+ assertEquals(1000, json.get("a"));
+ }
+
+ private ContainerId createContainerId(int id) {
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId containerId = ContainerId.newContainerId(appAttemptId, id);
+ return containerId;
+ }
+
+ @Test
+ public void testGetYarnGpuResourceInfo()
+ throws YarnException, InterruptedException, JSONException {
+ ResourcePluginManager rpm = mock(ResourcePluginManager.class);
+ Map<String, ResourcePlugin> namesToPlugins = new HashMap<>();
+ ResourcePlugin mockPlugin1 = mock(ResourcePlugin.class);
+ GpuDeviceInformation gpuDeviceInformation = new GpuDeviceInformation();
+ gpuDeviceInformation.setDriverVersion("1.2.3");
+ gpuDeviceInformation.setGpus(Arrays.asList(new PerGpuDeviceInformation()));
+ NMResourceInfo nmResourceInfo1 = new NMGpuResourceInfo(gpuDeviceInformation,
+ Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 2),
+ new GpuDevice(3, 3)), Arrays
+ .asList(new AssignedGpuDevice(2, 2, createContainerId(1)),
+ new AssignedGpuDevice(3, 3, createContainerId(2))));
+ when(mockPlugin1.getNMResourceInfo()).thenReturn(nmResourceInfo1);
+ namesToPlugins.put("resource-1", mockPlugin1);
+ namesToPlugins.put("yarn.io/resource-1", mockPlugin1);
+ ResourcePlugin mockPlugin2 = mock(ResourcePlugin.class);
+ namesToPlugins.put("resource-2", mockPlugin2);
+ when(rpm.getNameToPlugins()).thenReturn(namesToPlugins);
+
+ nmContext.setResourcePluginManager(rpm);
+
+ WebResource r = resource();
+ ClientResponse response;
+ JSONObject json;
+
+ // Access resource-1 should success
+ response = r.path("ws").path("v1").path("node").path(
+ "resources").path("resource-1").accept(MediaType.APPLICATION_JSON).get(
+ ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON, response.getType().toString());
+ json = response.getEntity(JSONObject.class);
+ System.out.println("GOT INFO " + json);
+ assertEquals("1.2.3",
+ json.getJSONObject("gpuDeviceInformation").get("driverVersion"));
+ assertEquals(3, json.getJSONArray("totalGpuDevices").length());
+ assertEquals(2, json.getJSONArray("assignedGpuDevices").length());
+ assertEquals(2, json.getJSONArray("assignedGpuDevices").length());
+ }
+
+ private void assertIncludesException(JSONObject json) {
+ assertTrue(json.has("RemoteException"));
+ }
+
private void testContainerLogs(WebResource r, ContainerId containerId)
throws IOException {
final String containerIdStr = containerId.toString();
@@ -587,7 +702,7 @@ public class TestNMWebServices extends JerseyTestBase {
}
// After container is completed, it is removed from nmContext
nmContext.getContainers().remove(containerId);
- Assert.assertNull(nmContext.getContainers().get(containerId));
+ assertNull(nmContext.getContainers().get(containerId));
response = r.path(filename).accept(MediaType.TEXT_PLAIN)
.get(ClientResponse.class);
responseText = response.getEntity(String.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java
index e22597d..dc96746 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java
@@ -39,7 +39,7 @@ public class TestGpuDeviceInformationParser {
Assert.assertEquals(2, info.getGpus().size());
PerGpuDeviceInformation gpu1 = info.getGpus().get(1);
Assert.assertEquals("Tesla P100-PCIE-12GB", gpu1.getProductName());
- Assert.assertEquals(16384, gpu1.getGpuMemoryUsage().getTotalMemoryMiB());
+ Assert.assertEquals(12193, gpu1.getGpuMemoryUsage().getTotalMemoryMiB());
Assert.assertEquals(10.3f,
gpu1.getGpuUtilizations().getOverallGpuUtilization(), 1e-6);
Assert.assertEquals(34f, gpu1.getTemperature().getCurrentGpuTemp(), 1e-6);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js
similarity index 70%
copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js
index d2937a0..bf6307a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js
@@ -16,9 +16,18 @@
* limitations under the License.
*/
-/**
- * Application level global constants go here.
- */
-export default {
- PARAM_SEPARATOR: '!',
-};
+import AbstractAdapter from './abstract';
+
+export default AbstractAdapter.extend({
+
+ address: "localBaseAddress",
+ restNameSpace: "node",
+ serverName: "NM",
+
+ urlForFindRecord(id/*, modelName, snapshot*/) {
+ var url = this._buildURL();
+ url = url.replace("{nodeAddress}", id) + "/resources/yarn.io%2Fgpu";
+ return url;
+ }
+
+});
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js
index ce26811..d476597 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js
@@ -20,6 +20,7 @@ import Ember from 'ember';
import BaseChartComponent from 'yarn-ui/components/base-chart-component';
import ColorUtils from 'yarn-ui/utils/color-utils';
import Converter from 'yarn-ui/utils/converter';
+import {Entities} from 'yarn-ui/constants';
export default BaseChartComponent.extend({
/*
@@ -41,8 +42,10 @@ export default BaseChartComponent.extend({
}
if (!middleValue) {
- if (this.get("type") === "memory") {
+ if (this.get(Entities.Type) === Entities.Memory) {
middleValue = Converter.memoryToSimpliedUnit(total);
+ } else if (this.get(Entities.Type) === Entities.Resource) {
+ middleValue = Converter.resourceToSimplifiedUnit(total, this.get(Entities.Unit));
} else {
middleValue = total;
}
@@ -151,7 +154,10 @@ export default BaseChartComponent.extend({
var value = d.value;
if (this.get("type") === "memory") {
value = Converter.memoryToSimpliedUnit(value);
+ } else if (this.get("type") === "resource") {
+ value = Converter.resourceToSimplifiedUnit(value, this.get(Entities.Unit));
}
+
return d.label + ' = ' + value + suffix;
}.bind(this));
}
@@ -184,11 +190,19 @@ export default BaseChartComponent.extend({
this.colors = ColorUtils.getColors(this.get("data").length, targets, colorTargetReverse);
}
- this.renderDonutChart(this.get("data"), this.get("title"), this.get("showLabels"),
- this.get("middleLabel"), this.get("middleValue"));
+ this.renderDonutChart(this.get("data"), this.get("title"), this.get("showLabels"),
+ this.get("middleLabel"), this.get("middleValue"), this.get("suffix"));
},
didInsertElement: function() {
+ // When parentIdPrefix is specified, use parentidPrefix + name as new parent
+ // id
+ if (this.get("parentIdPrefix")) {
+ var newParentId = this.get("parentIdPrefix") + this.get("id");
+ this.set("parentId", newParentId);
+ console.log(newParentId);
+ }
+
this.initChart();
this.draw();
},
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js
new file mode 100644
index 0000000..fa5ca8a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import DonutChart from 'yarn-ui/components/donut-chart';
+import ColorUtils from 'yarn-ui/utils/color-utils';
+
+export default DonutChart.extend({
+ draw: function() {
+ // Construct data
+ var data = [];
+ if (this.get("gpu-render-type") === "gpu-memory") {
+ data.push({
+ label: "Used",
+ value: parseFloat(this.get("gpuInfo").gpuMemoryUsage.usedMemoryMiB),
+ });
+ data.push({
+ label: "Available",
+ value: parseFloat(this.get("gpuInfo").gpuMemoryUsage.availMemoryMiB)
+ });
+ } else if (this.get("gpu-render-type") === "gpu-utilization") {
+ var utilization = parseFloat(this.get("gpuInfo").gpuUtilizations.overallGpuUtilization);
+ data.push({
+ label: "Utilized",
+ value: utilization,
+ });
+ data.push({
+ label: "Available",
+ value: 100 - utilization
+ });
+ }
+
+ var colorTargets = this.get("colorTargets");
+ if (colorTargets) {
+ var colorTargetReverse = Boolean(this.get("colorTargetReverse"));
+ var targets = colorTargets.split(" ");
+ this.colors = ColorUtils.getColors(data.length, targets, colorTargetReverse);
+ }
+
+ this.renderDonutChart(data, this.get("title"), this.get("showLabels"),
+ this.get("middleLabel"), this.get("middleValue"), this.get("suffix"));
+ },
+
+ didInsertElement: function() {
+ // ParentId includes minorNumber
+ var newParentId = this.get("parentId") + this.get("gpuInfo").minorNumber;
+ this.set("parentId", newParentId);
+
+ this.initChart();
+ this.draw();
+ },
+});
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
index d2937a0..29ad4bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
@@ -22,3 +22,16 @@
export default {
PARAM_SEPARATOR: '!',
};
+
+const BASE_UNIT = 1024
+
+export const Type = 'type';
+export const Memory = 'memory';
+export const Resource = 'resource';
+export const Unit = 'unit';
+export const Entities = {
+ Type: 'type',
+ Memory:'memory',
+ Resource: 'resource',
+ Unit: 'unit'
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
index 3fae596..f4bd578 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
@@ -60,7 +60,7 @@ export default Ember.Controller.extend({
getCellContent: function(row) {
var node_id = row.get("id"),
node_addr = row.get("nodeHTTPAddress"),
- href = `#/yarn-node/${node_id}/${node_addr}`;
+ href = `#/yarn-node/${node_id}/${node_addr}/info`;
switch(row.get("nodeState")) {
case "SHUTDOWN":
case "LOST":
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
index dcc0c29..d9a5eef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
@@ -43,6 +43,8 @@ export default DS.Model.extend({
decommissionedNodes: DS.attr('number'),
rebootedNodes: DS.attr('number'),
activeNodes: DS.attr('number'),
+ totalUsedResourcesAcrossPartition: DS.attr('object'),
+ totalClusterResourcesAcrossPartition: DS.attr('object'),
getFinishedAppsDataForDonutChart: function() {
var arr = [];
@@ -135,4 +137,71 @@ export default DS.Model.extend({
return arr;
}.property("allocatedVirtualCores", "reservedVirtualCores", "availableVirtualCores"),
+
+ getResourceTypes: function() {
+ var types = [];
+ if (this.get("totalClusterResourcesAcrossPartition")) {
+
+ console.log(types);
+ }
+ }.property("totalClusterResourcesAcrossPartition"),
+
+ /*
+ * Returned format
+ * [
+ * {
+ * name: <resource-name>
+ * unit: <resource-unit>
+ * [
+ * {
+ * label: <label>
+ * value: <value>
+ * },
+ * {
+ * }
+ * ...
+ * ],
+ * }
+ * ]
+ */
+ getAllResourceTypesDonutChart: function() {
+ if (this.get("totalClusterResourcesAcrossPartition")
+ && this.get("totalUsedResourcesAcrossPartition")) {
+ var usages = [];
+
+ var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourcesInformations;
+ var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourcesInformations;
+
+ clusterResourceInformations.forEach(function(cluster) {
+ var perResourceTypeUsage = {
+ name: cluster.name,
+ unit: cluster.units,
+ data: []
+ };
+
+ usedResourceInformations.forEach(function (used) {
+ if (used.name === perResourceTypeUsage.name) {
+ var usedValue = used.value;
+ perResourceTypeUsage.data.push({
+ label: "Used",
+ value: usedValue
+ }, {
+ label: "Available",
+ value: cluster.value - usedValue
+ });
+ }
+ });
+
+ usages.push(perResourceTypeUsage);
+
+ // Make sure id is a valid w3c ID
+ perResourceTypeUsage.id = perResourceTypeUsage.name.replace('/', '-');
+ perResourceTypeUsage.id = perResourceTypeUsage.id.replace('.', '-');
+ });
+
+ console.log(usages);
+ return usages;
+ }
+ return null;
+ }.property()
});
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-nm-gpu.js
similarity index 81%
copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-nm-gpu.js
index d2937a0..b3e9c2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-nm-gpu.js
@@ -16,9 +16,12 @@
* limitations under the License.
*/
-/**
- * Application level global constants go here.
- */
-export default {
- PARAM_SEPARATOR: '!',
-};
+import DS from 'ember-data';
+
+export default DS.Model.extend({
+ info: DS.attr('object'),
+
+ jsonString: function() {
+ return JSON.stringify(this.get("info"));
+ }.property(),
+});
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
index 1cb07bb..9b0f9ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
@@ -36,6 +36,7 @@ export default DS.Model.extend({
numActiveApplications: DS.attr('number'),
users: DS.hasMany('YarnUser'),
type: DS.attr('string'),
+ resources: DS.attr('object'),
isLeafQueue: function() {
var len = this.get("children.length");
@@ -91,5 +92,5 @@ export default DS.Model.extend({
value: this.get("numActiveApplications") || 0
}
];
- }.property("numPendingApplications", "numActiveApplications")
+ }.property("numPendingApplications", "numActiveApplications"),
});
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
index 20b6f5b..b1b1518 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
@@ -32,6 +32,8 @@ export default DS.Model.extend({
availableVirtualCores: DS.attr('number'),
version: DS.attr('string'),
nodeLabels: DS.attr('array'),
+ availableResource: DS.attr('object'),
+ usedResource: DS.attr('object'),
nodeLabelsAsString: function() {
var labels = this.get("nodeLabels");
@@ -90,6 +92,39 @@ export default DS.Model.extend({
return arr;
}.property("availableVirtualCores", "usedVirtualCores"),
+ getGpuDataForDonutChart: function() {
+ var arr = [];
+ var used = 0;
+ var ri;
+
+ var resourceInformations = this.get("usedResource").resourcesInformations;
+ for (var i = 0; i < resourceInformations.length; i++) {
+ ri = resourceInformations[i];
+ if (ri.name === "yarn.io/gpu") {
+ used = ri.value;
+ }
+ }
+
+ var available = 0;
+ resourceInformations = this.get("availableResource").resourcesInformations;
+ for (i = 0; i < resourceInformations.length; i++) {
+ ri = resourceInformations[i];
+ if (ri.name === "yarn.io/gpu") {
+ available = ri.value;
+ }
+ }
+
+ arr.push({
+ label: "Used",
+ value: used
+ });
+ arr.push({
+ label: "Available",
+ value: available
+ });
+ return arr;
+ }.property("availableResource", "usedResource"),
+
toolTipText: function() {
return "<p>Rack: " + this.get("rack") + '</p>' +
"<p>Host: " + this.get("nodeHostName") + '</p>';
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js
index 9013142..1a01b86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js
@@ -37,7 +37,10 @@ Router.map(function() {
this.route('apps');
});
this.route('yarn-nodes-heatmap');
- this.route('yarn-node', { path: '/yarn-node/:node_id/:node_addr' });
+ this.route('yarn-node', { path: '/yarn-node/:node_id/:node_addr' }, function() {
+ this.route("info");
+ this.route("yarn-nm-gpu");
+ });
this.route('yarn-node-apps', { path: '/yarn-node-apps/:node_id/:node_addr' });
this.route('yarn-node-app',
{ path: '/yarn-node-app/:node_id/:node_addr/:app_id' });
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js
index d03ea0d..254ece4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js
@@ -31,7 +31,7 @@ export default AbstractRoute.extend({
queues: this.store.query("yarn-queue.yarn-queue", {}).then((model) => {
let type = model.get('firstObject').get('type');
return this.store.query("yarn-queue." + type + "-queue", {});
- }),
+ })
});
},
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js
index 3d54846..7ce615c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js
@@ -25,6 +25,7 @@ export default AbstractRoute.extend({
// Fetches data from both NM and RM. RM is queried to get node usage info.
return Ember.RSVP.hash({
nodeInfo: { id: param.node_id, addr: param.node_addr },
+ nmGpuInfo: this.store.findRecord('yarn-nm-gpu', param.node_addr, {reload:true}),
node: this.store.findRecord('yarn-node', param.node_addr, {reload: true}),
rmNode: this.store.findRecord('yarn-rm-node', param.node_id, {reload: true})
});
@@ -33,5 +34,6 @@ export default AbstractRoute.extend({
unloadAll() {
this.store.unloadAll('yarn-node');
this.store.unloadAll('yarn-rm-node');
+ this.store.unloadAll('yarn-nm-gpu');
}
});
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node/yarn-nm-gpu.js
similarity index 89%
copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node/yarn-nm-gpu.js
index d2937a0..38ae5d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node/yarn-nm-gpu.js
@@ -16,9 +16,7 @@
* limitations under the License.
*/
-/**
- * Application level global constants go here.
- */
-export default {
- PARAM_SEPARATOR: '!',
-};
+import Ember from 'ember';
+
+export default Ember.Route.extend({
+});
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-nm-gpu.js
similarity index 56%
copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js
copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-nm-gpu.js
index 3d54846..3567c68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-nm-gpu.js
@@ -16,22 +16,28 @@
* limitations under the License.
*/
-import Ember from 'ember';
+import DS from 'ember-data';
-import AbstractRoute from './abstract';
+export default DS.JSONAPISerializer.extend({
+ internalNormalizeSingleResponse(store, primaryModelClass, payload, id) {
+ if (payload.nodeInfo) {
+ payload = payload.nodeInfo;
+ }
-export default AbstractRoute.extend({
- model(param) {
- // Fetches data from both NM and RM. RM is queried to get node usage info.
- return Ember.RSVP.hash({
- nodeInfo: { id: param.node_id, addr: param.node_addr },
- node: this.store.findRecord('yarn-node', param.node_addr, {reload: true}),
- rmNode: this.store.findRecord('yarn-rm-node', param.node_id, {reload: true})
- });
+ var fixedPayload = {
+ id: id,
+ type: primaryModelClass.modelName,
+ attributes: {
+ info: payload
+ }
+ };
+ return fixedPayload;
},
- unloadAll() {
- this.store.unloadAll('yarn-node');
- this.store.unloadAll('yarn-rm-node');
- }
+ normalizeSingleResponse(store, primaryModelClass, payload, id/*, requestType*/) {
+ // payload is of the form {"nodeInfo":{}}
+ var p = this.internalNormalizeSingleResponse(store,
+ primaryModelClass, payload, id);
+ return { data: p };
+ },
});
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
index c7350ef..7626598 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
@@ -72,6 +72,7 @@ export default DS.JSONAPISerializer.extend({
preemptionDisabled: payload.preemptionDisabled,
numPendingApplications: payload.numPendingApplications,
numActiveApplications: payload.numActiveApplications,
+ resources: payload.resources,
type: "capacity",
},
// Relationships
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
index 1c6d1be..a3a1d59 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
@@ -41,7 +41,9 @@ export default DS.JSONAPISerializer.extend({
usedVirtualCores: payload.usedVirtualCores,
availableVirtualCores: payload.availableVirtualCores,
version: payload.version,
- nodeLabels: payload.nodeLabels
+ nodeLabels: payload.nodeLabels,
+ usedResource: payload.used,
+ availableResource: payload.avail
}
};
return fixedPayload;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs
index e549ce5..ff4682a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs
@@ -90,41 +90,71 @@
<hr>
<div class="row">
-
- <div class="col-lg-4 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- Resource - Memory
+ <!-- When getAllResourceTypesDonutChart is not null, use it to show per-resource-type usages. Otherwise only show
+ vcore/memory usage from metrics -->
+ {{#if model.clusterMetrics.firstObject.getAllResourceTypesDonutChart}}
+ {{#each
+ model.clusterMetrics.firstObject.getAllResourceTypesDonutChart as |perTypeUsage|}}
+ <div class="col-lg-4 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ {{perTypeUsage.name}} - Usages
+ </div>
+ <div class="container-fluid" id="resource-type-{{perTypeUsage.id}}">
+ {{donut-chart
+ data=perTypeUsage.data
+ showLabels=true
+ parentIdPrefix="resource-type-"
+ id=perTypeUsage.id
+ ratio=0.6
+ unit=perTypeUsage.unit
+ type="resource"
+ maxHeight=350
+ colorTargets="good"
+ colorTargetReverse=true}}
+ </div>
+ </div>
</div>
- <div class="container-fluid" id="mem-donut-chart">
- {{donut-chart data=model.clusterMetrics.firstObject.getMemoryDataForDonutChart
- showLabels=true
- parentId="mem-donut-chart"
- ratio=0.6
- maxHeight=350
- colorTargets="good"
- colorTargetReverse=true
- type="memory"}}
+ {{/each}}
+ {{else}}
+ <div class="col-lg-4 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Resource - Memory
+ </div>
+ <div class="container-fluid" id="mem-donut-chart">
+ {{donut-chart
+ data=model.clusterMetrics.firstObject.getMemoryDataForDonutChart
+ showLabels=true
+ parentId="mem-donut-chart"
+ ratio=0.6
+ maxHeight=350
+ colorTargets="good"
+ colorTargetReverse=true
+ type="memory"}}
+ </div>
</div>
</div>
- </div>
- <div class="col-lg-4 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- Resource - VCores
- </div>
- <div class="container-fluid" id="vcore-donut-chart">
- {{donut-chart data=model.clusterMetrics.firstObject.getVCoreDataForDonutChart
- showLabels=true
- parentId="vcore-donut-chart"
- ratio=0.6
- maxHeight=350
- colorTargets="good"
- colorTargetReverse=true}}
+ <div class="col-lg-4 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Resource - VCores
+ </div>
+ <div class="container-fluid" id="vcore-donut-chart">
+ {{donut-chart
+ data=model.clusterMetrics.firstObject.getVCoreDataForDonutChart
+ showLabels=true
+ parentId="vcore-donut-chart"
+ ratio=0.6
+ maxHeight=350
+ colorTargets="good"
+ colorTargetReverse=true}}
+ </div>
</div>
</div>
- </div>
+ {{/if}}
+
</div>
<div class="row">
<div class="col-lg-6 container-fluid">
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs
index d2486c9..fffae30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs
@@ -24,8 +24,8 @@
<div class="panel-body">
<ul class="nav nav-pills nav-stacked" id="stacked-menu">
<ul class="nav nav-pills nav-stacked collapse in">
- {{#link-to 'yarn-node' tagName="li"}}
- {{#link-to 'yarn-node' nodeId nodeAddr}}Node Information
+ {{#link-to 'yarn-node.info' tagName="li"}}
+ {{#link-to 'yarn-node.info' nodeId nodeAddr}}Node Information
{{/link-to}}
{{/link-to}}
{{#link-to 'yarn-node-apps' tagName="li"}}
@@ -36,6 +36,12 @@
{{#link-to 'yarn-node-containers' nodeId nodeAddr}}List of Containers
{{/link-to}}
{{/link-to}}
+ {{#if nmGpuInfo}}
+ {{#link-to 'yarn-node.yarn-nm-gpu' tagName="li"}}
+ {{#link-to 'yarn-node.yarn-nm-gpu' nodeId nodeAddr }}GPU Information
+ {{/link-to}}
+ {{/link-to}}
+ {{/if}}
</ul>
</ul>
</div>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-nm-gpu-info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-nm-gpu-info.hbs
new file mode 100644
index 0000000..4118b1e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-nm-gpu-info.hbs
@@ -0,0 +1,69 @@
+{{!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+}}
+
+<div class="panel panel-default">
+ <div class="panel-heading">Gpu Information - (Minor
+ Number {{gpu.minorNumber}})
+ </div>
+ <table class="table">
+ <tbody>
+ <tr>
+ <td>Product Name</td>
+ <td>{{gpu.productName}}</td>
+ </tr>
+ <tr>
+ <td>UUID</td>
+ <td>{{gpu.uuid}}</td>
+ </tr>
+ <tr>
+ <td>Current Temperature</td>
+ <td>{{gpu.temperature.currentGpuTemp}}</td>
+ </tr>
+ <tr>
+ <td>Max Temperature</td>
+ <td>{{gpu.temperature.maxGpuTemp}}</td>
+ </tr>
+ </tbody>
+ </table>
+
+ <div class="col-md-5 container-fluid" id="mem-donut-chart{{gpu.minorNumber}}">
+ {{gpu-donut-chart gpuInfo=gpu
+ showLabels=true
+ parentId="mem-donut-chart"
+ middleLabel = "Gpu Memory"
+ ratio=0.6
+ type="memory"
+ gpu-render-type = "gpu-memory"
+ colorTargets="good"
+ colorTargetReverse=true
+ maxHeight=350}}
+ </div>
+
+ <div class="col-md-5 container-fluid" id="utilization-donut-chart{{gpu.minorNumber}}">
+ {{gpu-donut-chart gpuInfo=gpu
+ showLabels=true
+ parentId="utilization-donut-chart"
+ middleLabel = "Gpu Utilization"
+ ratio=0.6
+ gpu-render-type = "gpu-utilization"
+ colorTargets="good"
+ colorTargetReverse=true
+ suffix="%"
+ maxHeight=350}}
+ </div>
+</div>
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node.hbs
deleted file mode 100644
index 1e8549b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node.hbs
+++ /dev/null
@@ -1,125 +0,0 @@
-{{!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
---}}
-
-{{breadcrumb-bar breadcrumbs=breadcrumbs}}
-
-<div class="col-md-12 container-fluid">
- <div class="row">
-
- {{node-menu-panel path="yarn-node" nodeId=model.rmNode.id nodeAddr=model.node.id}}
-
- <div class="col-md-10 container-fluid">
-
- <div class="row">
- <div class="col-md-12 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">Node Information: {{model.rmNode.id}}</div>
- <table class="table">
- <tbody>
- <tr>
- <td>Total Vmem allocated for Containers</td>
- <td>{{divide num=model.node.totalVmemAllocatedContainersMB den=1024}} GB</td>
- </tr>
- <tr>
- <td>Vmem enforcement enabled</td>
- <td>{{model.node.vmemCheckEnabled}}</td>
- </tr>
- <tr>
- <td>Total Pmem allocated for Containers</td>
- <td>{{divide num=model.node.totalPmemAllocatedContainersMB den=1024}} GB</td>
- </tr>
- <tr>
- <td>Pmem enforcement enabled</td>
- <td>{{model.node.pmemCheckEnabled}}</td>
- </tr>
- <tr>
- <td>Total VCores allocated for Containers</td>
- <td>{{model.node.totalVCoresAllocatedContainers}}</td>
- </tr>
- <tr>
- <td>Node Healthy Status</td>
- <td>{{model.node.nodeHealthy}}</td>
- </tr>
- <tr>
- <td>Last Node Health Report Time</td>
- <td>{{model.node.lastNodeUpdateTime}}</td>
- </tr>
- <tr>
- <td>Node Health Report</td>
- <td>{{model.node.healthReport}}</td>
- </tr>
- {{#if model.node.nmStartupTime}}
- <tr>
- <td>Node Manager Start Time</td>
- <td>{{model.node.nmStartupTime}}</td>
- </tr>
- {{/if}}
- <tr>
- <td>Node Manager Version</td>
- <td>{{model.node.nodeManagerBuildVersion}}</td>
- </tr>
- <tr>
- <td>Hadoop Version</td>
- <td>{{model.node.hadoopBuildVersion}}</td>
- </tr>
- </tbody>
- </table>
- </div>
- </div>
- </div>
-
- <div class="row">
- <div class="col-lg-6 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- Resource - Memory
- </div>
- <div class="container-fluid" id="mem-donut-chart">
- {{donut-chart data=model.rmNode.getMemoryDataForDonutChart
- showLabels=true
- parentId="mem-donut-chart"
- ratio=0.6
- type="memory"
- colorTargets="good"
- colorTargetReverse=true
- maxHeight=350}}
- </div>
- </div>
- </div>
-
- <div class="col-lg-6 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- Resource - VCores
- </div>
- <div class="container-fluid" id="vcore-donut-chart">
- {{donut-chart data=model.rmNode.getVCoreDataForDonutChart
- showLabels=true
- parentId="vcore-donut-chart"
- ratio=0.6
- colorTargets="good"
- colorTargetReverse=true
- maxHeight=350}}
- </div>
- </div>
- </div>
- </div>
- </div>
- </div>
-</div>
-{{outlet}}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/info.hbs
new file mode 100644
index 0000000..ad411c0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/info.hbs
@@ -0,0 +1,154 @@
+{{!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--}}
+
+{{breadcrumb-bar breadcrumbs=breadcrumbs}}
+
+<div class="col-md-12 container-fluid">
+ <div class="row">
+
+ {{node-menu-panel path="yarn-node" nodeId=model.rmNode.id
+ nodeAddr=model.node.id nmGpuInfo=model.nmGpuInfo}}
+
+ <div class="col-md-10 container-fluid">
+
+ <div class="row">
+ <div class="col-md-12 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">Node
+ Information: {{model.rmNode.id}}</div>
+ <table class="table">
+ <tbody>
+ <tr>
+ <td>Total Vmem allocated for Containers</td>
+ <td>{{divide num=model.node.totalVmemAllocatedContainersMB
+ den=1024}} GB
+ </td>
+ </tr>
+ <tr>
+ <td>Vmem enforcement enabled</td>
+ <td>{{model.node.vmemCheckEnabled}}</td>
+ </tr>
+ <tr>
+ <td>Total Pmem allocated for Containers</td>
+ <td>{{divide num=model.node.totalPmemAllocatedContainersMB
+ den=1024}} GB
+ </td>
+ </tr>
+ <tr>
+ <td>Pmem enforcement enabled</td>
+ <td>{{model.node.pmemCheckEnabled}}</td>
+ </tr>
+ <tr>
+ <td>Total VCores allocated for Containers</td>
+ <td>{{model.node.totalVCoresAllocatedContainers}}</td>
+ </tr>
+ <tr>
+ <td>Node Healthy Status</td>
+ <td>{{model.node.nodeHealthy}}</td>
+ </tr>
+ <tr>
+ <td>Last Node Health Report Time</td>
+ <td>{{model.node.lastNodeUpdateTime}}</td>
+ </tr>
+ <tr>
+ <td>Node Health Report</td>
+ <td>{{model.node.healthReport}}</td>
+ </tr>
+ {{#if model.node.nmStartupTime}}
+ <tr>
+ <td>Node Manager Start Time</td>
+ <td>{{model.node.nmStartupTime}}</td>
+ </tr>
+ {{/if}}
+ <tr>
+ <td>Node Manager Version</td>
+ <td>{{model.node.nodeManagerBuildVersion}}</td>
+ </tr>
+ <tr>
+ <td>Hadoop Version</td>
+ <td>{{model.node.hadoopBuildVersion}}</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+ </div>
+
+ <div class="row">
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Resource - Memory
+ </div>
+ <div class="container-fluid" id="mem-donut-chart">
+ {{donut-chart data=model.rmNode.getMemoryDataForDonutChart
+ showLabels=true
+ parentId="mem-donut-chart"
+ ratio=0.6
+ type="memory"
+ colorTargets="good"
+ colorTargetReverse=true
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Resource - VCores
+ </div>
+ <div class="container-fluid" id="vcore-donut-chart">
+ {{donut-chart data=model.rmNode.getVCoreDataForDonutChart
+ showLabels=true
+ parentId="vcore-donut-chart"
+ ratio=0.6
+ colorTargets="good"
+ colorTargetReverse=true
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+ </div>
+
+ {{#if model.nmGpuInfo}}
+ <div class="row">
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ <li>
+ Resources - yarn.io/gpu
+ </li>
+ </div>
+ <div class="container-fluid" id="gpu-donut-chart">
+ {{donut-chart data=model.rmNode.getGpuDataForDonutChart
+ showLabels=true
+ parentId="gpu-donut-chart"
+ ratio=0.6
+ colorTargets="good"
+ colorTargetReverse=true
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+ </div>
+ {{/if}}
+ </div>
+ </div>
+</div>
+{{outlet}}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs
new file mode 100644
index 0000000..55840ad
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs
@@ -0,0 +1,53 @@
+{{!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--}}
+
+{{breadcrumb-bar breadcrumbs=breadcrumbs}}
+
+<div class="col-md-12 container-fluid">
+ <div class="row">
+
+ {{node-menu-panel path="yarn-node" nodeId=model.rmNode.id
+ nodeAddr=model.node.id nmGpuInfo=model.nmGpuInfo}}
+
+ <div class="col-md-10 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">Gpu Information</div>
+ <table class="table">
+ <tbody>
+ <tr>
+ <td>Vendor</td>
+ <td>NVIDIA</td>
+ </tr>
+ <tr>
+ <td>Driver Version</td>
+ <td>{{model.nmGpuInfo.info.gpuDeviceInformation.driverVersion}}</td>
+ </tr>
+ <tr>
+ <td>Total Number Of Gpus</td>
+ <td>{{model.nmGpuInfo.info.totalGpuDevices.length}}</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+
+ {{#each model.nmGpuInfo.info.gpuDeviceInformation.gpus as |gpu|}}
+ {{yarn-nm-gpu-info gpu=gpu}}
+ {{/each}}
+ </div>
+ </div>
+</div>
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js
index 7c9a1f8..e47edad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js
@@ -130,6 +130,57 @@ export default {
}
return value.toFixed(1) + " " + unit;
},
+ resourceToSimplifiedUnit: function (value, unit) {
+ // First convert unit to base unit ("").
+ var normalizedValue = value;
+ if (unit === "Ki") {
+ normalizedValue = normalizedValue * 1024;
+ } else if (unit === "Mi") {
+ normalizedValue = normalizedValue * 1024 * 1024;
+ } else if (unit === "Gi") {
+ normalizedValue = normalizedValue * 1024 * 1024 * 1024;
+ } else if (unit === "Ti") {
+ normalizedValue = normalizedValue * 1024 * 1024 * 1024 * 1024;
+ } else if (unit === "Pi") {
+ normalizedValue = normalizedValue * 1024 * 1024 * 1024 * 1024 * 1024;
+ } else if (unit === "K" || unit === "k") {
+ normalizedValue = normalizedValue * 1000;
+ } else if (unit === "M" || unit === "m") {
+ normalizedValue = normalizedValue * 1000 * 1000;
+ } else if (unit === "G" || unit === "g") {
+ normalizedValue = normalizedValue * 1000 * 1000 * 1000;
+ } else if (unit === "T" || unit === "t") {
+ normalizedValue = normalizedValue * 1000 * 1000 * 1000 * 1000;
+ } else if (unit === "P" || unit === "p") {
+ normalizedValue = normalizedValue * 1000 * 1000 * 1000 * 1000 * 1000;
+ }
+
+ // From baseunit ("") convert to most human readable unit
+ // (which value < 1024 * 0.9).
+ var finalUnit = "";
+ if (normalizedValue / 1024 >= 0.9) {
+ normalizedValue = normalizedValue / 1024;
+ finalUnit = "Ki";
+ }
+ if (normalizedValue / 1024 >= 0.9) {
+ normalizedValue = normalizedValue / 1024;
+ finalUnit = "Mi";
+ }
+ if (normalizedValue / 1024 >= 0.9) {
+ normalizedValue = normalizedValue / 1024;
+ finalUnit = "Gi";
+ }
+ if (normalizedValue / 1024 >= 0.9) {
+ normalizedValue = normalizedValue / 1024;
+ finalUnit = "Ti";
+ }
+ if (normalizedValue / 1024 >= 0.9) {
+ normalizedValue = normalizedValue / 1024;
+ finalUnit = "Pi";
+ }
+
+ return normalizedValue.toFixed(1) + " " + finalUnit;
+ },
msToElapsedTimeUnit: function(millisecs, short) {
var seconds = Math.floor(millisecs / 1000);
var days = Math.floor(seconds / (3600 * 24));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 06/09: YARN-7594. TestNMWebServices#testGetNMResourceInfo fails on trunk. Contributed by Gergely Novák.
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit c3e4d9f3e466102bf7aa9d04a9a879edb116eb31
Author: Sunil G <su...@apache.org>
AuthorDate: Mon Dec 4 10:45:07 2017 +0530
YARN-7594. TestNMWebServices#testGetNMResourceInfo fails on trunk. Contributed by Gergely Novák.
---
.../yarn/server/nodemanager/webapp/TestNMWebServices.java | 12 ++++--------
1 file changed, 4 insertions(+), 8 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
index c9c86b7..8cd8777 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
@@ -444,17 +444,17 @@ public class TestNMWebServices extends JerseyTestBase {
ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON, response.getType().toString());
- // Access resource-2 should fail (null NMResourceInfo returned).
+ // Access resource-2 should fail (empty NMResourceInfo returned).
JSONObject json = response.getEntity(JSONObject.class);
- assertIncludesException(json);
+ assertEquals(0, json.length());
- // Access resource-3 should fail (unkown plugin)
+ // Access resource-3 should fail (unknown plugin)
response = r.path("ws").path("v1").path("node").path(
"resources").path("resource-3").accept(MediaType.APPLICATION_JSON).get(
ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON, response.getType().toString());
json = response.getEntity(JSONObject.class);
- assertIncludesException(json);
+ assertEquals(0, json.length());
// Access resource-1 should success
response = r.path("ws").path("v1").path("node").path(
@@ -522,10 +522,6 @@ public class TestNMWebServices extends JerseyTestBase {
assertEquals(2, json.getJSONArray("assignedGpuDevices").length());
}
- private void assertIncludesException(JSONObject json) {
- assertTrue(json.has("RemoteException"));
- }
-
private void testContainerLogs(WebResource r, ContainerId containerId)
throws IOException {
final String containerIdStr = containerId.toString();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 08/09: YARN-7143. FileNotFound handling in ResourceUtils
is inconsistent
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit e4ede15388f5f280be5e58a36997b22b82ff73af
Author: Daniel Templeton <te...@apache.org>
AuthorDate: Thu Nov 9 10:36:49 2017 -0800
YARN-7143. FileNotFound handling in ResourceUtils is inconsistent
Change-Id: Ib1bb487e14a15edd2b5a42cf5078c5a2b295f069
(cherry picked from commit db82a41d94872cea4d0c1bb1336916cebc2faeec)
---
.../hadoop/yarn/util/resource/ResourceUtils.java | 52 +++++++++-------------
1 file changed, 22 insertions(+), 30 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index f3edc74..abf58a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -338,18 +338,14 @@ public class ResourceUtils {
if (!initializedResources) {
synchronized (ResourceUtils.class) {
if (!initializedResources) {
- if (conf == null) {
- conf = new YarnConfiguration();
- }
- try {
- addResourcesFileToConf(resourceFile, conf);
- LOG.debug("Found " + resourceFile + ", adding to configuration");
- } catch (FileNotFoundException fe) {
- LOG.info("Unable to find '" + resourceFile
- + "'. Falling back to memory and vcores as resources.");
+ Configuration resConf = conf;
+
+ if (resConf == null) {
+ resConf = new YarnConfiguration();
}
- initializeResourcesMap(conf);
+ addResourcesFileToConf(resourceFile, resConf);
+ initializeResourcesMap(resConf);
}
}
}
@@ -386,21 +382,17 @@ public class ResourceUtils {
}
private static void addResourcesFileToConf(String resourceFile,
- Configuration conf) throws FileNotFoundException {
+ Configuration conf) {
try {
InputStream ris = getConfInputStream(resourceFile, conf);
LOG.debug("Found " + resourceFile + ", adding to configuration");
conf.addResource(ris);
} catch (FileNotFoundException fe) {
- throw fe;
- } catch (IOException ie) {
+ LOG.info("Unable to find '" + resourceFile + "'.");
+ } catch (IOException | YarnException ex) {
LOG.fatal("Exception trying to read resource types configuration '"
- + resourceFile + "'.", ie);
- throw new YarnRuntimeException(ie);
- } catch (YarnException ye) {
- LOG.fatal("YARN Exception trying to read resource types configuration '"
- + resourceFile + "'.", ye);
- throw new YarnRuntimeException(ye);
+ + resourceFile + "'.", ex);
+ throw new YarnRuntimeException(ex);
}
}
@@ -462,19 +454,19 @@ public class ResourceUtils {
private static Map<String, ResourceInformation> initializeNodeResourceInformation(
Configuration conf) {
Map<String, ResourceInformation> nodeResources = new HashMap<>();
- try {
- addResourcesFileToConf(
- YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE, conf);
- for (Map.Entry<String, String> entry : conf) {
- String key = entry.getKey();
- String value = entry.getValue();
- if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
- addResourceInformation(key, value, nodeResources);
- }
+
+ addResourcesFileToConf(YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE,
+ conf);
+
+ for (Map.Entry<String, String> entry : conf) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+
+ if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
+ addResourceInformation(key, value, nodeResources);
}
- } catch (FileNotFoundException fe) {
- LOG.info("Couldn't find node resources file");
}
+
return nodeResources;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/09: Partial commit of YARN-7224 to get GpuDevice class
and changes
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit e9ce7165957dd6dba4d71e625a1ed91f175fb8e6
Author: Jonathan Hung <jh...@linkedin.com>
AuthorDate: Thu Apr 12 13:56:16 2018 -0700
Partial commit of YARN-7224 to get GpuDevice class and changes
---
.../nodemanager/DockerContainerExecutor.java | 2 +-
.../container/ResourceMappings.java | 2 +-
.../linux/resources/gpu/GpuResourceAllocator.java | 102 ++++++-------
.../resources/gpu/GpuResourceHandlerImpl.java | 26 ++--
.../resourceplugin/gpu/GpuDevice.java | 78 ++++++++++
.../resourceplugin/gpu/GpuDiscoverer.java | 30 ++--
.../gpu/GpuNodeResourceUpdateHandler.java | 10 +-
.../recovery/NMLeveldbStateStoreService.java | 68 +++++----
.../recovery/NMNullStateStoreService.java | 5 +-
.../nodemanager/recovery/NMStateStoreService.java | 17 ++-
.../TestDockerContainerExecutorWithMocks.java | 2 +-
.../TestContainerManagerRecovery.java | 15 +-
.../resources/gpu/TestGpuResourceHandler.java | 158 ++++++++++++++-------
.../resourceplugin/gpu/TestGpuDiscoverer.java | 34 +++--
.../recovery/NMMemoryStateStoreService.java | 10 +-
.../recovery/TestNMLeveldbStateStoreService.java | 28 +++-
16 files changed, 395 insertions(+), 192 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
index a044cb6..6c2eb96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
@@ -117,7 +117,7 @@ public class DockerContainerExecutor extends ContainerExecutor {
}
@Override
- public void init() throws IOException {
+ public void init(Context nmContext) throws IOException {
String auth =
getConf().get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
if (auth != null && !auth.equals("simple")) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ResourceMappings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ResourceMappings.java
index d673341..225192b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ResourceMappings.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ResourceMappings.java
@@ -84,7 +84,7 @@ public class ResourceMappings {
return Collections.unmodifiableList(resources);
}
- public void updateAssignedResources(List<Serializable> list) {
+ public void updateAssignedResources(List<? extends Serializable> list) {
this.resources = new ArrayList<>(list);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
index d6bae09..f2bb342 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
@@ -26,12 +26,11 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import java.io.IOException;
import java.io.Serializable;
@@ -54,8 +53,8 @@ import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI;
public class GpuResourceAllocator {
final static Log LOG = LogFactory.getLog(GpuResourceAllocator.class);
- private Set<Integer> allowedGpuDevices = new TreeSet<>();
- private Map<Integer, ContainerId> usedDevices = new TreeMap<>();
+ private Set<GpuDevice> allowedGpuDevices = new TreeSet<>();
+ private Map<GpuDevice, ContainerId> usedDevices = new TreeMap<>();
private Context nmContext;
public GpuResourceAllocator(Context ctx) {
@@ -63,14 +62,14 @@ public class GpuResourceAllocator {
}
/**
- * Contains allowed and denied devices with minor number.
+ * Contains allowed and denied devices
* Denied devices will be useful for cgroups devices module to do blacklisting
*/
static class GpuAllocation {
- private Set<Integer> allowed = Collections.emptySet();
- private Set<Integer> denied = Collections.emptySet();
+ private Set<GpuDevice> allowed = Collections.emptySet();
+ private Set<GpuDevice> denied = Collections.emptySet();
- GpuAllocation(Set<Integer> allowed, Set<Integer> denied) {
+ GpuAllocation(Set<GpuDevice> allowed, Set<GpuDevice> denied) {
if (allowed != null) {
this.allowed = ImmutableSet.copyOf(allowed);
}
@@ -79,21 +78,21 @@ public class GpuResourceAllocator {
}
}
- public Set<Integer> getAllowedGPUs() {
+ public Set<GpuDevice> getAllowedGPUs() {
return allowed;
}
- public Set<Integer> getDeniedGPUs() {
+ public Set<GpuDevice> getDeniedGPUs() {
return denied;
}
}
/**
* Add GPU to allowed list
- * @param minorNumber minor number of the GPU device.
+ * @param gpuDevice gpu device
*/
- public synchronized void addGpu(int minorNumber) {
- allowedGpuDevices.add(minorNumber);
+ public synchronized void addGpu(GpuDevice gpuDevice) {
+ allowedGpuDevices.add(gpuDevice);
}
private String getResourceHandlerExceptionMessage(int numRequestedGpuDevices,
@@ -117,42 +116,42 @@ public class GpuResourceAllocator {
+ containerId);
}
- for (Serializable deviceId : c.getResourceMappings().getAssignedResources(
- GPU_URI)){
- if (!(deviceId instanceof String)) {
+ for (Serializable gpuDeviceSerializable : c.getResourceMappings()
+ .getAssignedResources(GPU_URI)) {
+ if (!(gpuDeviceSerializable instanceof GpuDevice)) {
throw new ResourceHandlerException(
"Trying to recover device id, however it"
- + " is not String, this shouldn't happen");
+ + " is not GpuDevice, this shouldn't happen");
}
-
- int devId;
- try {
- devId = Integer.parseInt((String)deviceId);
- } catch (NumberFormatException e) {
- throw new ResourceHandlerException("Failed to recover device id because"
- + "it is not a valid integer, devId:" + deviceId);
- }
+ GpuDevice gpuDevice = (GpuDevice) gpuDeviceSerializable;
// Make sure it is in allowed GPU device.
- if (!allowedGpuDevices.contains(devId)) {
- throw new ResourceHandlerException("Try to recover device id = " + devId
- + " however it is not in allowed device list:" + StringUtils
- .join(",", allowedGpuDevices));
+ if (!allowedGpuDevices.contains(gpuDevice)) {
+ throw new ResourceHandlerException(
+ "Try to recover device = " + gpuDevice
+ + " however it is not in allowed device list:" + StringUtils
+ .join(",", allowedGpuDevices));
}
// Make sure it is not occupied by anybody else
- if (usedDevices.containsKey(devId)) {
- throw new ResourceHandlerException("Try to recover device id = " + devId
- + " however it is already assigned to container=" + usedDevices
- .get(devId) + ", please double check what happened.");
+ if (usedDevices.containsKey(gpuDevice)) {
+ throw new ResourceHandlerException(
+ "Try to recover device id = " + gpuDevice
+ + " however it is already assigned to container=" + usedDevices
+ .get(gpuDevice) + ", please double check what happened.");
}
- usedDevices.put(devId, containerId);
+ usedDevices.put(gpuDevice, containerId);
}
}
- private int getRequestedGpus(Resource requestedResource) {
+ /**
+ * Get number of requested GPUs from resource.
+ * @param requestedResource requested resource
+ * @return #gpus.
+ */
+ public static int getRequestedGpus(Resource requestedResource) {
try {
return Long.valueOf(requestedResource.getResourceValue(
GPU_URI)).intValue();
@@ -164,8 +163,8 @@ public class GpuResourceAllocator {
/**
* Assign GPU to requestor
* @param container container to allocate
- * @return List of denied Gpus with minor numbers
- * @throws ResourceHandlerException When failed to
+ * @return allocation results.
+ * @throws ResourceHandlerException When failed to assign GPUs.
*/
public synchronized GpuAllocation assignGpus(Container container)
throws ResourceHandlerException {
@@ -180,12 +179,12 @@ public class GpuResourceAllocator {
containerId));
}
- Set<Integer> assignedGpus = new HashSet<>();
+ Set<GpuDevice> assignedGpus = new TreeSet<>();
- for (int deviceNum : allowedGpuDevices) {
- if (!usedDevices.containsKey(deviceNum)) {
- usedDevices.put(deviceNum, containerId);
- assignedGpus.add(deviceNum);
+ for (GpuDevice gpu : allowedGpuDevices) {
+ if (!usedDevices.containsKey(gpu)) {
+ usedDevices.put(gpu, containerId);
+ assignedGpus.add(gpu);
if (assignedGpus.size() == numRequestedGpuDevices) {
break;
}
@@ -194,21 +193,10 @@ public class GpuResourceAllocator {
// Record in state store if we allocated anything
if (!assignedGpus.isEmpty()) {
- List<Serializable> allocatedDevices = new ArrayList<>();
- for (int gpu : assignedGpus) {
- allocatedDevices.add(String.valueOf(gpu));
- }
try {
- // Update Container#getResourceMapping.
- ResourceMappings.AssignedResources assignedResources =
- new ResourceMappings.AssignedResources();
- assignedResources.updateAssignedResources(allocatedDevices);
- container.getResourceMappings().addAssignedResources(GPU_URI,
- assignedResources);
-
// Update state store.
- nmContext.getNMStateStore().storeAssignedResources(containerId,
- GPU_URI, allocatedDevices);
+ nmContext.getNMStateStore().storeAssignedResources(container, GPU_URI,
+ new ArrayList<>(assignedGpus));
} catch (IOException e) {
cleanupAssignGpus(containerId);
throw new ResourceHandlerException(e);
@@ -226,7 +214,7 @@ public class GpuResourceAllocator {
* @param containerId containerId
*/
public synchronized void cleanupAssignGpus(ContainerId containerId) {
- Iterator<Map.Entry<Integer, ContainerId>> iter =
+ Iterator<Map.Entry<GpuDevice, ContainerId>> iter =
usedDevices.entrySet().iterator();
while (iter.hasNext()) {
if (iter.next().getValue().equals(containerId)) {
@@ -236,7 +224,7 @@ public class GpuResourceAllocator {
}
@VisibleForTesting
- public synchronized Map<Integer, ContainerId> getDeviceAllocationMapping() {
+ public synchronized Map<GpuDevice, ContainerId> getDeviceAllocationMapping() {
return new HashMap<>(usedDevices);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
index 7144bb2..4a783d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
@@ -24,8 +24,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -35,6 +33,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDiscoverer;
import java.util.ArrayList;
@@ -64,17 +63,23 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
@Override
public List<PrivilegedOperation> bootstrap(Configuration configuration)
throws ResourceHandlerException {
- List<Integer> minorNumbersOfUsableGpus;
+ List<GpuDevice> usableGpus;
try {
- minorNumbersOfUsableGpus = GpuDiscoverer.getInstance()
- .getMinorNumbersOfGpusUsableByYarn();
+ usableGpus = GpuDiscoverer.getInstance()
+ .getGpusUsableByYarn();
+ if (usableGpus == null || usableGpus.isEmpty()) {
+ String message = "GPU is enabled on the NodeManager, but couldn't find "
+ + "any usable GPU devices, please double check configuration.";
+ LOG.error(message);
+ throw new ResourceHandlerException(message);
+ }
} catch (YarnException e) {
LOG.error("Exception when trying to get usable GPU device", e);
throw new ResourceHandlerException(e);
}
- for (int minorNumber : minorNumbersOfUsableGpus) {
- gpuAllocator.addGpu(minorNumber);
+ for (GpuDevice gpu : usableGpus) {
+ gpuAllocator.addGpu(gpu);
}
// And initialize cgroups
@@ -102,10 +107,13 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
PrivilegedOperation.OperationType.GPU, Arrays
.asList(CONTAINER_ID_CLI_OPTION, containerIdStr));
if (!allocation.getDeniedGPUs().isEmpty()) {
+ List<Integer> minorNumbers = new ArrayList<>();
+ for (GpuDevice deniedGpu : allocation.getDeniedGPUs()) {
+ minorNumbers.add(deniedGpu.getMinorNumber());
+ }
privilegedOperation.appendArgs(Arrays.asList(EXCLUDED_GPUS_CLI_OPTION,
- StringUtils.join(",", allocation.getDeniedGPUs())));
+ StringUtils.join(",", minorNumbers)));
}
-
privilegedOperationExecutor.executePrivilegedOperation(
privilegedOperation, true);
} catch (PrivilegedOperationException e) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java
new file mode 100644
index 0000000..8119924
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu;
+
+import java.io.Serializable;
+
+/**
+ * This class is used to represent GPU device while allocation.
+ */
+public class GpuDevice implements Serializable, Comparable {
+ private int index;
+ private int minorNumber;
+ private static final long serialVersionUID = -6812314470754667710L;
+
+ public GpuDevice(int index, int minorNumber) {
+ this.index = index;
+ this.minorNumber = minorNumber;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+ public int getMinorNumber() {
+ return minorNumber;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || !(obj instanceof GpuDevice)) {
+ return false;
+ }
+ GpuDevice other = (GpuDevice) obj;
+ return index == other.index && minorNumber == other.minorNumber;
+ }
+
+ @Override
+ public int compareTo(Object obj) {
+ if (obj == null || (!(obj instanceof GpuDevice))) {
+ return -1;
+ }
+
+ GpuDevice other = (GpuDevice) obj;
+
+ int result = Integer.compare(index, other.index);
+ if (0 != result) {
+ return result;
+ }
+ return Integer.compare(minorNumber, other.minorNumber);
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 47;
+ return prime * index + minorNumber;
+ }
+
+ @Override
+ public String toString() {
+ return "(index=" + index + ",minor_number=" + minorNumber + ")";
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java
index 61b8ce5..6e3cf13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDiscoverer.java
@@ -136,12 +136,12 @@ public class GpuDiscoverer {
}
/**
- * Get list of minor device numbers of Gpu devices usable by YARN.
+ * Get list of GPU devices usable by YARN.
*
- * @return List of minor device numbers of Gpu devices.
+ * @return List of GPU devices
* @throws YarnException when any issue happens
*/
- public synchronized List<Integer> getMinorNumbersOfGpusUsableByYarn()
+ public synchronized List<GpuDevice> getGpusUsableByYarn()
throws YarnException {
validateConfOrThrowException();
@@ -149,7 +149,7 @@ public class GpuDiscoverer {
YarnConfiguration.NM_GPU_ALLOWED_DEVICES,
YarnConfiguration.AUTOMATICALLY_DISCOVER_GPU_DEVICES);
- List<Integer> minorNumbers = new ArrayList<>();
+ List<GpuDevice> gpuDevices = new ArrayList<>();
if (allowedDevicesStr.equals(
YarnConfiguration.AUTOMATICALLY_DISCOVER_GPU_DEVICES)) {
@@ -167,21 +167,31 @@ public class GpuDiscoverer {
}
if (lastDiscoveredGpuInformation.getGpus() != null) {
- for (PerGpuDeviceInformation gpu : lastDiscoveredGpuInformation
- .getGpus()) {
- minorNumbers.add(gpu.getMinorNumber());
+ for (int i = 0; i < lastDiscoveredGpuInformation.getGpus().size();
+ i++) {
+ List<PerGpuDeviceInformation> gpuInfos =
+ lastDiscoveredGpuInformation.getGpus();
+ gpuDevices.add(new GpuDevice(i, gpuInfos.get(i).getMinorNumber()));
}
}
} else{
for (String s : allowedDevicesStr.split(",")) {
if (s.trim().length() > 0) {
- minorNumbers.add(Integer.valueOf(s.trim()));
+ String[] kv = s.trim().split(":");
+ if (kv.length != 2) {
+ throw new YarnException(
+ "Illegal format, it should be index:minor_number format, now it="
+ + s);
+ }
+
+ gpuDevices.add(
+ new GpuDevice(Integer.parseInt(kv[0]), Integer.parseInt(kv[1])));
}
}
- LOG.info("Allowed GPU devices with minor numbers:" + allowedDevicesStr);
+ LOG.info("Allowed GPU devices:" + gpuDevices);
}
- return minorNumbers;
+ return gpuDevices;
}
public synchronized void initialize(Configuration conf) throws YarnException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java
index f6bf506..796eb25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeResourceUpdateHandler.java
@@ -40,12 +40,14 @@ public class GpuNodeResourceUpdateHandler extends NodeResourceUpdaterPlugin {
public void updateConfiguredResource(Resource res) throws YarnException {
LOG.info("Initializing configured GPU resources for the NodeManager.");
- List<Integer> usableGpus =
- GpuDiscoverer.getInstance().getMinorNumbersOfGpusUsableByYarn();
+ List<GpuDevice> usableGpus =
+ GpuDiscoverer.getInstance().getGpusUsableByYarn();
if (null == usableGpus || usableGpus.isEmpty()) {
- LOG.info("Didn't find any usable GPUs on the NodeManager.");
+ String message = "GPU is enabled, but couldn't find any usable GPUs on the "
+ + "NodeManager.";
+ LOG.error(message);
// No gpu can be used by YARN.
- return;
+ throw new YarnException(message);
}
long nUsableGpus = usableGpus.size();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index 6aec1be..98d6d02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -18,28 +18,9 @@
package org.apache.hadoop.yarn.server.nodemanager.recovery;
-import static org.fusesource.leveldbjni.JniDBFactory.asString;
-import static org.fusesource.leveldbjni.JniDBFactory.bytes;
-
-import org.slf4j.Logger;
-import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.Set;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ListMultimap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -51,9 +32,11 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestP
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
+import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ContainerTokenIdentifierProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto;
@@ -61,9 +44,10 @@ import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.Deletion
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto;
-import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ContainerTokenIdentifierProto;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
@@ -76,10 +60,26 @@ import org.iq80.leveldb.DB;
import org.iq80.leveldb.DBException;
import org.iq80.leveldb.Options;
import org.iq80.leveldb.WriteBatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import static org.fusesource.leveldbjni.JniDBFactory.asString;
+import static org.fusesource.leveldbjni.JniDBFactory.bytes;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ListMultimap;
public class NMLeveldbStateStoreService extends NMStateStoreService {
@@ -1180,15 +1180,18 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
}
@Override
- public void storeAssignedResources(ContainerId containerId,
- String resourceType, List<Serializable> assignedResources)
+ public void storeAssignedResources(Container container,
+ String resourceType, List<? extends Serializable> assignedResources)
throws IOException {
if (LOG.isDebugEnabled()) {
- LOG.debug("storeAssignedResources: containerId=" + containerId
- + ", assignedResources=" + StringUtils.join(",", assignedResources));
+ LOG.debug(
+ "storeAssignedResources: containerId=" + container.getContainerId()
+ + ", assignedResources=" + StringUtils
+ .join(",", assignedResources));
+
}
- String keyResChng = CONTAINERS_KEY_PREFIX + containerId.toString()
+ String keyResChng = CONTAINERS_KEY_PREFIX + container.getContainerId().toString()
+ CONTAINER_ASSIGNED_RESOURCES_KEY_SUFFIX + resourceType;
try {
WriteBatch batch = db.createWriteBatch();
@@ -1206,6 +1209,9 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
} catch (DBException e) {
throw new IOException(e);
}
+
+ // update container resource mapping.
+ updateContainerResourceMapping(container, resourceType, assignedResources);
}
@SuppressWarnings("deprecation")
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
index 6e3707b..77ed241 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.Localize
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
// The state store to use when state isn't being stored
public class NMNullStateStoreService extends NMStateStoreService {
@@ -268,8 +269,8 @@ public class NMNullStateStoreService extends NMStateStoreService {
}
@Override
- public void storeAssignedResources(ContainerId containerId,
- String resourceType, List<Serializable> assignedResources)
+ public void storeAssignedResources(Container container,
+ String resourceType, List<? extends Serializable> assignedResources)
throws IOException {
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
index a929fe2..44f5e3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.Localize
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
@Private
@@ -732,13 +733,13 @@ public abstract class NMStateStoreService extends AbstractService {
/**
* Store the assigned resources to a container.
*
- * @param containerId Container Id
+ * @param container NMContainer
* @param resourceType Resource Type
* @param assignedResources Assigned resources
* @throws IOException if fails
*/
- public abstract void storeAssignedResources(ContainerId containerId,
- String resourceType, List<Serializable> assignedResources)
+ public abstract void storeAssignedResources(Container container,
+ String resourceType, List<? extends Serializable> assignedResources)
throws IOException;
protected abstract void initStorage(Configuration conf) throws IOException;
@@ -746,4 +747,14 @@ public abstract class NMStateStoreService extends AbstractService {
protected abstract void startStorage() throws IOException;
protected abstract void closeStorage() throws IOException;
+
+ protected void updateContainerResourceMapping(Container container,
+ String resourceType, List<? extends Serializable> assignedResources) {
+ // Update Container#getResourceMapping.
+ ResourceMappings.AssignedResources newAssigned =
+ new ResourceMappings.AssignedResources();
+ newAssigned.updateAssignedResources(assignedResources);
+ container.getResourceMappings().addAssignedResources(resourceType,
+ newAssigned);
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java
index f1194c9..d1f126e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java
@@ -116,7 +116,7 @@ public class TestDockerContainerExecutorWithMocks {
public void testContainerInitSecure() throws IOException {
dockerContainerExecutor.getConf().set(
CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- dockerContainerExecutor.init();
+ dockerContainerExecutor.init(null);
}
@Test(expected = IllegalArgumentException.class)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index 4e48366..b56daa2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -519,16 +519,18 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
commonLaunchContainer(appId, cid, cm);
+ Container nmContainer = context.getContainers().get(cid);
+
Application app = context.getApplications().get(appId);
assertNotNull(app);
// store resource mapping of the container
- List<Serializable> gpuResources = Arrays.asList("1", "2", "3");
- stateStore.storeAssignedResources(cid, "gpu", gpuResources);
- List<Serializable> numaResources = Arrays.asList("numa1");
- stateStore.storeAssignedResources(cid, "numa", numaResources);
- List<Serializable> fpgaResources = Arrays.asList("fpga1", "fpga2");
- stateStore.storeAssignedResources(cid, "fpga", fpgaResources);
+ List<String> gpuResources = Arrays.asList("1", "2", "3");
+ stateStore.storeAssignedResources(nmContainer, "gpu", gpuResources);
+ List<String> numaResources = Arrays.asList("numa1");
+ stateStore.storeAssignedResources(nmContainer, "numa", numaResources);
+ List<String> fpgaResources = Arrays.asList("fpga1", "fpga2");
+ stateStore.storeAssignedResources(nmContainer, "fpga", fpgaResources);
cm.stop();
context = createContext(conf, stateStore);
@@ -540,7 +542,6 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
app = context.getApplications().get(appId);
assertNotNull(app);
- Container nmContainer = context.getContainers().get(cid);
Assert.assertNotNull(nmContainer);
ResourceMappings resourceMappings = nmContainer.getResourceMappings();
List<Serializable> assignedResource = resourceMappings
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
index 5c70f7a..a4ac4bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resourc
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -36,15 +35,17 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDiscoverer;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
-import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -90,7 +91,7 @@ public class TestGpuResourceHandler {
@Test
public void testBootStrap() throws Exception {
Configuration conf = new YarnConfiguration();
- conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0");
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0");
GpuDiscoverer.getInstance().initialize(conf);
@@ -104,8 +105,8 @@ public class TestGpuResourceHandler {
.newInstance(ApplicationId.newInstance(1234L, 1), 1), id);
}
- private static Container mockContainerWithGpuRequest(int id,
- int numGpuRequest) {
+ private static Container mockContainerWithGpuRequest(int id, int numGpuRequest,
+ boolean dockerContainerEnabled) {
Container c = mock(Container.class);
when(c.getContainerId()).thenReturn(getContainerId(id));
@@ -115,29 +116,46 @@ public class TestGpuResourceHandler {
res.setResourceValue(ResourceInformation.GPU_URI, numGpuRequest);
when(c.getResource()).thenReturn(res);
when(c.getResourceMappings()).thenReturn(resMapping);
+
+ ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
+ Map<String, String> env = new HashMap<>();
+ if (dockerContainerEnabled) {
+ env.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker");
+ }
+ when(clc.getEnvironment()).thenReturn(env);
+ when(c.getLaunchContext()).thenReturn(clc);
return c;
}
+ private static Container mockContainerWithGpuRequest(int id,
+ int numGpuRequest) {
+ return mockContainerWithGpuRequest(id, numGpuRequest, false);
+ }
+
private void verifyDeniedDevices(ContainerId containerId,
- List<Integer> deniedDevices)
+ List<GpuDevice> deniedDevices)
throws ResourceHandlerException, PrivilegedOperationException {
verify(mockCGroupsHandler, times(1)).createCGroup(
CGroupsHandler.CGroupController.DEVICES, containerId.toString());
if (null != deniedDevices && !deniedDevices.isEmpty()) {
+ List<Integer> deniedDevicesMinorNumber = new ArrayList<>();
+ for (GpuDevice deniedDevice : deniedDevices) {
+ deniedDevicesMinorNumber.add(deniedDevice.getMinorNumber());
+ }
verify(mockPrivilegedExecutor, times(1)).executePrivilegedOperation(
new PrivilegedOperation(PrivilegedOperation.OperationType.GPU, Arrays
.asList(GpuResourceHandlerImpl.CONTAINER_ID_CLI_OPTION,
containerId.toString(),
GpuResourceHandlerImpl.EXCLUDED_GPUS_CLI_OPTION,
- StringUtils.join(",", deniedDevices))), true);
+ StringUtils.join(",", deniedDevicesMinorNumber))), true);
}
}
- @Test
- public void testAllocation() throws Exception {
+ private void commonTestAllocation(boolean dockerContainerEnabled)
+ throws Exception {
Configuration conf = new YarnConfiguration();
- conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0,1,3,4");
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
GpuDiscoverer.getInstance().initialize(conf);
gpuResourceHandler.bootstrap(conf);
@@ -145,31 +163,52 @@ public class TestGpuResourceHandler {
gpuResourceHandler.getGpuAllocator().getAvailableGpus());
/* Start container 1, asks 3 containers */
- gpuResourceHandler.preStart(mockContainerWithGpuRequest(1, 3));
+ gpuResourceHandler.preStart(
+ mockContainerWithGpuRequest(1, 3, dockerContainerEnabled));
// Only device=4 will be blocked.
- verifyDeniedDevices(getContainerId(1), Arrays.asList(4));
+ if (dockerContainerEnabled) {
+ verifyDeniedDevices(getContainerId(1), Collections.<GpuDevice>emptyList());
+ } else{
+ verifyDeniedDevices(getContainerId(1), Arrays.asList(new GpuDevice(3,4)));
+ }
/* Start container 2, asks 2 containers. Excepted to fail */
boolean failedToAllocate = false;
try {
- gpuResourceHandler.preStart(mockContainerWithGpuRequest(2, 2));
+ gpuResourceHandler.preStart(
+ mockContainerWithGpuRequest(2, 2, dockerContainerEnabled));
} catch (ResourceHandlerException e) {
failedToAllocate = true;
}
Assert.assertTrue(failedToAllocate);
/* Start container 3, ask 1 container, succeeded */
- gpuResourceHandler.preStart(mockContainerWithGpuRequest(3, 1));
+ gpuResourceHandler.preStart(
+ mockContainerWithGpuRequest(3, 1, dockerContainerEnabled));
// devices = 0/1/3 will be blocked
- verifyDeniedDevices(getContainerId(3), Arrays.asList(0, 1, 3));
+ if (dockerContainerEnabled) {
+ verifyDeniedDevices(getContainerId(3), Collections.<GpuDevice>emptyList());
+ } else {
+ verifyDeniedDevices(getContainerId(3), Arrays
+ .asList(new GpuDevice(0, 0), new GpuDevice(1, 1),
+ new GpuDevice(2, 3)));
+ }
- /* Start container 4, ask 0 container, succeeded */
- gpuResourceHandler.preStart(mockContainerWithGpuRequest(4, 0));
- // All devices will be blocked
- verifyDeniedDevices(getContainerId(4), Arrays.asList(0, 1, 3, 4));
+ /* Start container 4, ask 0 container, succeeded */
+ gpuResourceHandler.preStart(
+ mockContainerWithGpuRequest(4, 0, dockerContainerEnabled));
+
+ if (dockerContainerEnabled) {
+ verifyDeniedDevices(getContainerId(4), Collections.<GpuDevice>emptyList());
+ } else{
+ // All devices will be blocked
+ verifyDeniedDevices(getContainerId(4), Arrays
+ .asList(new GpuDevice(0, 0), new GpuDevice(1, 1), new GpuDevice(2, 3),
+ new GpuDevice(3, 4)));
+ }
/* Release container-1, expect cgroups deleted */
gpuResourceHandler.postComplete(getContainerId(1));
@@ -188,12 +227,24 @@ public class TestGpuResourceHandler {
gpuResourceHandler.getGpuAllocator().getAvailableGpus());
}
+ @Test
+ public void testAllocationWhenDockerContainerEnabled() throws Exception {
+ // When docker container is enabled, no devices should be written to
+ // devices.deny.
+ commonTestAllocation(true);
+ }
+
+ @Test
+ public void testAllocation() throws Exception {
+ commonTestAllocation(false);
+ }
+
@SuppressWarnings("unchecked")
@Test
public void testAssignedGpuWillBeCleanedupWhenStoreOpFails()
throws Exception {
Configuration conf = new YarnConfiguration();
- conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0,1,3,4");
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
GpuDiscoverer.getInstance().initialize(conf);
gpuResourceHandler.bootstrap(conf);
@@ -202,7 +253,7 @@ public class TestGpuResourceHandler {
doThrow(new IOException("Exception ...")).when(mockNMStateStore)
.storeAssignedResources(
- any(ContainerId.class), anyString(), anyList());
+ any(Container.class), anyString(), anyList());
boolean exception = false;
/* Start container 1, asks 3 containers */
@@ -225,13 +276,16 @@ public class TestGpuResourceHandler {
conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, " ");
GpuDiscoverer.getInstance().initialize(conf);
- gpuResourceHandler.bootstrap(conf);
- Assert.assertEquals(0,
- gpuResourceHandler.getGpuAllocator().getAvailableGpus());
+ try {
+ gpuResourceHandler.bootstrap(conf);
+ Assert.fail("Should fail because no GPU available");
+ } catch (ResourceHandlerException e) {
+ // Expected because of no resource available
+ }
/* Start container 1, asks 0 containers */
gpuResourceHandler.preStart(mockContainerWithGpuRequest(1, 0));
- verifyDeniedDevices(getContainerId(1), Collections.emptyList());
+ verifyDeniedDevices(getContainerId(1), Collections.<GpuDevice>emptyList());
/* Start container 2, asks 1 containers. Excepted to fail */
boolean failedToAllocate = false;
@@ -254,7 +308,7 @@ public class TestGpuResourceHandler {
@Test
public void testAllocationStored() throws Exception {
Configuration conf = new YarnConfiguration();
- conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0,1,3,4");
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
GpuDiscoverer.getInstance().initialize(conf);
gpuResourceHandler.bootstrap(conf);
@@ -265,33 +319,33 @@ public class TestGpuResourceHandler {
Container container = mockContainerWithGpuRequest(1, 3);
gpuResourceHandler.preStart(container);
- verify(mockNMStateStore).storeAssignedResources(getContainerId(1),
- ResourceInformation.GPU_URI,
- Arrays.asList("0", "1", "3"));
-
- Assert.assertEquals(3, container.getResourceMappings()
- .getAssignedResources(ResourceInformation.GPU_URI).size());
+ verify(mockNMStateStore).storeAssignedResources(container,
+ ResourceInformation.GPU_URI, Arrays
+ .asList(new GpuDevice(0, 0), new GpuDevice(1, 1),
+ new GpuDevice(2, 3)));
// Only device=4 will be blocked.
- verifyDeniedDevices(getContainerId(1), Arrays.asList(4));
+ verifyDeniedDevices(getContainerId(1), Arrays.<GpuDevice>asList(new GpuDevice(3, 4)));
/* Start container 2, ask 0 container, succeeded */
container = mockContainerWithGpuRequest(2, 0);
gpuResourceHandler.preStart(container);
- verifyDeniedDevices(getContainerId(2), Arrays.asList(0, 1, 3, 4));
+ verifyDeniedDevices(getContainerId(2), Arrays
+ .asList(new GpuDevice(0, 0), new GpuDevice(1, 1), new GpuDevice(2, 3),
+ new GpuDevice(3, 4)));
Assert.assertEquals(0, container.getResourceMappings()
.getAssignedResources(ResourceInformation.GPU_URI).size());
// Store assigned resource will not be invoked.
verify(mockNMStateStore, never()).storeAssignedResources(
- eq(getContainerId(2)), eq(ResourceInformation.GPU_URI), anyList());
+ eq(container), eq(ResourceInformation.GPU_URI), anyList());
}
@Test
public void testRecoverResourceAllocation() throws Exception {
Configuration conf = new YarnConfiguration();
- conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0,1,3,4");
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
GpuDiscoverer.getInstance().initialize(conf);
gpuResourceHandler.bootstrap(conf);
@@ -302,7 +356,8 @@ public class TestGpuResourceHandler {
ResourceMappings rmap = new ResourceMappings();
ResourceMappings.AssignedResources ar =
new ResourceMappings.AssignedResources();
- ar.updateAssignedResources(Arrays.asList("1", "3"));
+ ar.updateAssignedResources(
+ Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 3)));
rmap.addAssignedResources(ResourceInformation.GPU_URI, ar);
when(nmContainer.getResourceMappings()).thenReturn(rmap);
@@ -312,12 +367,15 @@ public class TestGpuResourceHandler {
// Reacquire container restore state of GPU Resource Allocator.
gpuResourceHandler.reacquireContainer(getContainerId(1));
- Map<Integer, ContainerId> deviceAllocationMapping =
+ Map<GpuDevice, ContainerId> deviceAllocationMapping =
gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
Assert.assertEquals(2, deviceAllocationMapping.size());
Assert.assertTrue(
- deviceAllocationMapping.keySet().containsAll(Arrays.asList(1, 3)));
- Assert.assertEquals(deviceAllocationMapping.get(1), getContainerId(1));
+ deviceAllocationMapping.keySet().contains(new GpuDevice(1, 1)));
+ Assert.assertTrue(
+ deviceAllocationMapping.keySet().contains(new GpuDevice(2, 3)));
+ Assert.assertEquals(deviceAllocationMapping.get(new GpuDevice(1, 1)),
+ getContainerId(1));
// TEST CASE
// Try to reacquire a container but requested device is not in allowed list.
@@ -325,7 +383,8 @@ public class TestGpuResourceHandler {
rmap = new ResourceMappings();
ar = new ResourceMappings.AssignedResources();
// id=5 is not in allowed list.
- ar.updateAssignedResources(Arrays.asList("4", "5"));
+ ar.updateAssignedResources(
+ Arrays.asList(new GpuDevice(3, 4), new GpuDevice(4, 5)));
rmap.addAssignedResources(ResourceInformation.GPU_URI, ar);
when(nmContainer.getResourceMappings()).thenReturn(rmap);
@@ -345,9 +404,10 @@ public class TestGpuResourceHandler {
deviceAllocationMapping =
gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
Assert.assertEquals(2, deviceAllocationMapping.size());
- Assert.assertTrue(
- deviceAllocationMapping.keySet().containsAll(Arrays.asList(1, 3)));
- Assert.assertEquals(deviceAllocationMapping.get(1), getContainerId(1));
+ Assert.assertTrue(deviceAllocationMapping.keySet()
+ .containsAll(Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 3))));
+ Assert.assertEquals(deviceAllocationMapping.get(new GpuDevice(1, 1)),
+ getContainerId(1));
// TEST CASE
// Try to reacquire a container but requested device is already assigned.
@@ -355,7 +415,8 @@ public class TestGpuResourceHandler {
rmap = new ResourceMappings();
ar = new ResourceMappings.AssignedResources();
// id=3 is already assigned
- ar.updateAssignedResources(Arrays.asList("4", "3"));
+ ar.updateAssignedResources(
+ Arrays.asList(new GpuDevice(3, 4), new GpuDevice(2, 3)));
rmap.addAssignedResources("gpu", ar);
when(nmContainer.getResourceMappings()).thenReturn(rmap);
@@ -375,8 +436,9 @@ public class TestGpuResourceHandler {
deviceAllocationMapping =
gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
Assert.assertEquals(2, deviceAllocationMapping.size());
- Assert.assertTrue(
- deviceAllocationMapping.keySet().containsAll(Arrays.asList(1, 3)));
- Assert.assertEquals(deviceAllocationMapping.get(1), getContainerId(1));
+ Assert.assertTrue(deviceAllocationMapping.keySet()
+ .containsAll(Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 3))));
+ Assert.assertEquals(deviceAllocationMapping.get(new GpuDevice(1, 1)),
+ getContainerId(1));
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuDiscoverer.java
index 83bace2..4abb633 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuDiscoverer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestGpuDiscoverer.java
@@ -101,23 +101,41 @@ public class TestGpuDiscoverer {
GpuDeviceInformation info = plugin.getGpuDeviceInformation();
Assert.assertTrue(info.getGpus().size() > 0);
- Assert.assertEquals(plugin.getMinorNumbersOfGpusUsableByYarn().size(),
+ Assert.assertEquals(plugin.getGpusUsableByYarn().size(),
info.getGpus().size());
}
@Test
public void getNumberOfUsableGpusFromConfig() throws YarnException {
Configuration conf = new Configuration(false);
- conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0,1,2,4");
+
+ // Illegal format
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:2,3");
GpuDiscoverer plugin = new GpuDiscoverer();
+ try {
+ plugin.initialize(conf);
+ plugin.getGpusUsableByYarn();
+ Assert.fail("Illegal format, should fail.");
+ } catch (YarnException e) {
+ // Expected
+ }
+
+ // Valid format
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:2,3:4");
+ plugin = new GpuDiscoverer();
plugin.initialize(conf);
- List<Integer> minorNumbers = plugin.getMinorNumbersOfGpusUsableByYarn();
- Assert.assertEquals(4, minorNumbers.size());
+ List<GpuDevice> usableGpuDevices = plugin.getGpusUsableByYarn();
+ Assert.assertEquals(4, usableGpuDevices.size());
+
+ Assert.assertTrue(0 == usableGpuDevices.get(0).getIndex());
+ Assert.assertTrue(1 == usableGpuDevices.get(1).getIndex());
+ Assert.assertTrue(2 == usableGpuDevices.get(2).getIndex());
+ Assert.assertTrue(3 == usableGpuDevices.get(3).getIndex());
- Assert.assertTrue(0 == minorNumbers.get(0));
- Assert.assertTrue(1 == minorNumbers.get(1));
- Assert.assertTrue(2 == minorNumbers.get(2));
- Assert.assertTrue(4 == minorNumbers.get(3));
+ Assert.assertTrue(0 == usableGpuDevices.get(0).getMinorNumber());
+ Assert.assertTrue(1 == usableGpuDevices.get(1).getMinorNumber());
+ Assert.assertTrue(2 == usableGpuDevices.get(2).getMinorNumber());
+ Assert.assertTrue(4 == usableGpuDevices.get(3).getMinorNumber());
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
index 5d424ad..95fe845 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDelet
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
@@ -515,14 +516,17 @@ public class NMMemoryStateStoreService extends NMStateStoreService {
}
@Override
- public void storeAssignedResources(ContainerId containerId,
- String resourceType, List<Serializable> assignedResources)
+ public void storeAssignedResources(Container container,
+ String resourceType, List<? extends Serializable> assignedResources)
throws IOException {
ResourceMappings.AssignedResources ar =
new ResourceMappings.AssignedResources();
ar.updateAssignedResources(assignedResources);
- containerStates.get(containerId).getResourceMappings()
+ containerStates.get(container.getContainerId()).getResourceMappings()
.addAssignedResources(resourceType, ar);
+
+ // update container resource mapping.
+ updateContainerResourceMapping(container, resourceType, assignedResources);
}
private static class TrackerState {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
index 7a6157d..750e1fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
@@ -29,6 +29,7 @@ import static org.mockito.Mockito.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
@@ -69,6 +70,8 @@ import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDelet
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.nodemanager.amrmproxy.AMRMProxyTokenSecretManager;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.LocalResourceTrackerState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredAMRMProxyState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredApplicationsState;
@@ -1143,16 +1146,21 @@ public class TestNMLeveldbStateStoreService {
ContainerId containerId = ContainerId.newContainerId(appAttemptId, 5);
storeMockContainer(containerId);
+ Container container = mock(Container.class);
+ when(container.getContainerId()).thenReturn(containerId);
+ ResourceMappings resourceMappings = new ResourceMappings();
+ when(container.getResourceMappings()).thenReturn(resourceMappings);
+
// Store ResourceMapping
- stateStore.storeAssignedResources(containerId, "gpu",
+ stateStore.storeAssignedResources(container, "gpu",
Arrays.asList("1", "2", "3"));
// This will overwrite above
- List<Serializable> gpuRes1 = Arrays.asList("1", "2", "4");
- stateStore.storeAssignedResources(containerId, "gpu", gpuRes1);
- List<Serializable> fpgaRes = Arrays.asList("3", "4", "5", "6");
- stateStore.storeAssignedResources(containerId, "fpga", fpgaRes);
- List<Serializable> numaRes = Arrays.asList("numa1");
- stateStore.storeAssignedResources(containerId, "numa", numaRes);
+ List<String> gpuRes1 = Arrays.asList("1", "2", "4");
+ stateStore.storeAssignedResources(container, "gpu", gpuRes1);
+ List<String> fpgaRes = Arrays.asList("3", "4", "5", "6");
+ stateStore.storeAssignedResources(container, "fpga", fpgaRes);
+ List<String> numaRes = Arrays.asList("numa1");
+ stateStore.storeAssignedResources(container, "numa", numaRes);
// add a invalid key
restartStateStore();
@@ -1162,12 +1170,18 @@ public class TestNMLeveldbStateStoreService {
List<Serializable> res = rcs.getResourceMappings()
.getAssignedResources("gpu");
Assert.assertTrue(res.equals(gpuRes1));
+ Assert.assertTrue(
+ resourceMappings.getAssignedResources("gpu").equals(gpuRes1));
res = rcs.getResourceMappings().getAssignedResources("fpga");
Assert.assertTrue(res.equals(fpgaRes));
+ Assert.assertTrue(
+ resourceMappings.getAssignedResources("fpga").equals(fpgaRes));
res = rcs.getResourceMappings().getAssignedResources("numa");
Assert.assertTrue(res.equals(numaRes));
+ Assert.assertTrue(
+ resourceMappings.getAssignedResources("numa").equals(numaRes));
}
private StartContainerRequest storeMockContainer(ContainerId containerId)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 04/09: YARN-7573. Gpu Information page could be empty for
nodes without GPU. (Sunil G via wangda)
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit b69e841d258099e46bd17943b97fdf2f38483fcd
Author: Wangda Tan <wa...@apache.org>
AuthorDate: Wed Nov 29 17:43:37 2017 -0800
YARN-7573. Gpu Information page could be empty for nodes without GPU. (Sunil G via wangda)
Change-Id: I7f614e5a589a09ce4e4286c84b706e05c29abd14
---
.../apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java | 4 +---
.../hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js | 6 ++++--
.../src/main/webapp/app/templates/components/node-menu-panel.hbs | 2 +-
.../hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-apps.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-node-containers.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs | 4 ++++
6 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index 7476d75..7702004 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -510,9 +510,7 @@ public class NMWebServices {
}
}
- throw new YarnException(
- "Could not get detailed resource information for given resource-name="
- + resourceName);
+ return new NMResourceInfo();
}
private long parseLongParam(String bytes) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
index b1b1518..aa5efbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
@@ -97,7 +97,9 @@ export default DS.Model.extend({
var used = 0;
var ri;
- var resourceInformations = this.get("usedResource").resourcesInformations;
+ const usedResource = this.get("usedResource");
+ const availableResource = this.get("availableResource");
+ var resourceInformations = usedResource ? usedResource.resourcesInformations : [];
for (var i = 0; i < resourceInformations.length; i++) {
ri = resourceInformations[i];
if (ri.name === "yarn.io/gpu") {
@@ -106,7 +108,7 @@ export default DS.Model.extend({
}
var available = 0;
- resourceInformations = this.get("availableResource").resourcesInformations;
+ resourceInformations = availableResource ? availableResource.resourcesInformations : [];
for (i = 0; i < resourceInformations.length; i++) {
ri = resourceInformations[i];
if (ri.name === "yarn.io/gpu") {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs
index fffae30..966e408 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs
@@ -36,7 +36,7 @@
{{#link-to 'yarn-node-containers' nodeId nodeAddr}}List of Containers
{{/link-to}}
{{/link-to}}
- {{#if nmGpuInfo}}
+ {{#if (and nmGpuInfo nmGpuInfo.info.totalGpuDevices)}}
{{#link-to 'yarn-node.yarn-nm-gpu' tagName="li"}}
{{#link-to 'yarn-node.yarn-nm-gpu' nodeId nodeAddr }}GPU Information
{{/link-to}}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-apps.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-apps.hbs
index 52f0c86..919e54d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-apps.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-apps.hbs
@@ -20,7 +20,7 @@
<div class="col-md-12 container-fluid">
<div class="row">
- {{node-menu-panel path="yarn-node-apps" nodeAddr=model.nodeInfo.addr nodeId=model.nodeInfo.id}}
+ {{node-menu-panel path="yarn-node-apps" nodeAddr=model.nodeInfo.addr nodeId=model.nodeInfo.id nmGpuInfo=model.nmGpuInfo}}
{{#if model.apps}}
<div class="col-md-10 container-fluid">
<table id="node-apps-table" class="display table table-striped table-bordered" cellspacing="0" width="100%">
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-containers.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-containers.hbs
index f520c46..1f31272 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-containers.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-containers.hbs
@@ -20,7 +20,7 @@
<div class="col-md-12 container-fluid">
<div class="row">
- {{node-menu-panel path="yarn-node-containers" nodeAddr=model.nodeInfo.addr nodeId=model.nodeInfo.id}}
+ {{node-menu-panel path="yarn-node-containers" nodeAddr=model.nodeInfo.addr nodeId=model.nodeInfo.id nmGpuInfo=model.nmGpuInfo}}
{{#if model.containers}}
<div class="col-md-10 container-fluid">
<table id="node-containers-table" class="display table table-striped table-bordered" cellspacing="0" width="100%">
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs
index 55840ad..0464cc8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs
@@ -23,6 +23,7 @@
{{node-menu-panel path="yarn-node" nodeId=model.rmNode.id
nodeAddr=model.node.id nmGpuInfo=model.nmGpuInfo}}
+ {{#if model.nmGpuInfo.info.totalGpuDevices}}
<div class="col-md-10 container-fluid">
<div class="panel panel-default">
@@ -49,5 +50,8 @@
{{yarn-nm-gpu-info gpu=gpu}}
{{/each}}
</div>
+ {{else}}
+ <h4 align = "center">No GPUs are found on this node.</h4>
+ {{/if}}
</div>
</div>
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 03/09: YARN-7345. GPU Isolation: Incorrect minor device
numbers written to devices.deny file. (Jonathan Hung via wangda)
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit e4268da636018d71f019823efe3c339ad481ef64
Author: Wangda Tan <wa...@apache.org>
AuthorDate: Thu Oct 19 14:45:44 2017 -0700
YARN-7345. GPU Isolation: Incorrect minor device numbers written to devices.deny file. (Jonathan Hung via wangda)
---
.../native/container-executor/impl/modules/gpu/gpu-module.c | 2 +-
.../container-executor/test/modules/gpu/test-gpu-module.cc | 13 +++++++++++++
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
index f96645d..1a1b164 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
@@ -108,7 +108,7 @@ static int internal_handle_gpu_request(
char param_value[128];
memset(param_value, 0, sizeof(param_value));
snprintf(param_value, sizeof(param_value), "c %d:%d rwm",
- major_device_number, i);
+ major_device_number, minor_devices[i]);
int rc = update_cgroups_parameters_func_p("devices", "deny",
container_id, param_value);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
index 7e41fb4..b3d93dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
@@ -165,6 +165,19 @@ TEST_F(TestGpuModule, test_verify_gpu_module_calls_cgroup_parameter) {
// Verify cgroups parameters
verify_param_updated_to_cgroups(0, NULL);
+
+ /* Test case 3: block 2 non-sequential devices */
+ cgroups_parameters_invoked.clear();
+ char* argv_2[] = { (char*) "--module-gpu", (char*) "--excluded_gpus", (char*) "1,3",
+ (char*) "--container_id", container_id };
+ rc = handle_gpu_request(&mock_update_cgroups_parameters,
+ "gpu", 5, argv_2);
+ ASSERT_EQ(0, rc) << "Should success.\n";
+
+ // Verify cgroups parameters
+ const char* expected_cgroups_argv_2[] = { "devices", "deny", container_id, "c 195:1 rwm",
+ "devices", "deny", container_id, "c 195:3 rwm"};
+ verify_param_updated_to_cgroups(8, expected_cgroups_argv_2);
}
TEST_F(TestGpuModule, test_illegal_cli_parameters) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 09/09: YARN-7383. Node resource is not parsed correctly for resource names containing dot. Contributed by Gergely Novák.
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit f76f2cf493cb56bc13a18f0da847778c4f1082d7
Author: Sunil G <su...@apache.org>
AuthorDate: Wed Dec 13 22:00:07 2017 +0530
YARN-7383. Node resource is not parsed correctly for resource names containing dot. Contributed by Gergely Novák.
---
.../apache/hadoop/yarn/util/resource/ResourceUtils.java | 15 ++++++---------
.../hadoop/yarn/util/resource/TestResourceUtils.java | 5 ++++-
.../test/resources/resource-types/node-resources-2.xml | 5 +++++
.../test/resources/resource-types/resource-types-4.xml | 7 ++++++-
4 files changed, 21 insertions(+), 11 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index abf58a6..65eb5a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -461,21 +461,18 @@ public class ResourceUtils {
for (Map.Entry<String, String> entry : conf) {
String key = entry.getKey();
String value = entry.getValue();
-
- if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
- addResourceInformation(key, value, nodeResources);
- }
+ addResourceTypeInformation(key, value, nodeResources);
}
return nodeResources;
}
- private static void addResourceInformation(String prop, String value,
+ private static void addResourceTypeInformation(String prop, String value,
Map<String, ResourceInformation> nodeResources) {
- String[] parts = prop.split("\\.");
- LOG.info("Found resource entry " + prop);
- if (parts.length == 4) {
- String resourceType = parts[3];
+ if (prop.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
+ LOG.info("Found resource entry " + prop);
+ String resourceType = prop.substring(
+ YarnConfiguration.NM_RESOURCES_PREFIX.length());
if (!nodeResources.containsKey(resourceType)) {
nodeResources
.put(resourceType, ResourceInformation.newInstance(resourceType));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
index 80555ca..b511705 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
@@ -124,9 +124,10 @@ public class TestResourceUtils {
new ResourceFileInformation("resource-types-3.xml", 3);
testFile3.resourceNameUnitsMap.put("resource2", "");
ResourceFileInformation testFile4 =
- new ResourceFileInformation("resource-types-4.xml", 4);
+ new ResourceFileInformation("resource-types-4.xml", 5);
testFile4.resourceNameUnitsMap.put("resource1", "G");
testFile4.resourceNameUnitsMap.put("resource2", "m");
+ testFile4.resourceNameUnitsMap.put("yarn.io/gpu", "");
ResourceFileInformation[] tests = {testFile1, testFile2, testFile3,
testFile4};
@@ -292,6 +293,8 @@ public class TestResourceUtils {
ResourceInformation.newInstance("resource1", "Gi", 5L));
test3Resources.setResourceInformation("resource2",
ResourceInformation.newInstance("resource2", "m", 2L));
+ test3Resources.setResourceInformation("yarn.io/gpu",
+ ResourceInformation.newInstance("yarn.io/gpu", "", 1));
testRun.put("node-resources-2.xml", test3Resources);
for (Map.Entry<String, Resource> entry : testRun.entrySet()) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml
index 9d9b3dc..382d5dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml
@@ -36,4 +36,9 @@ limitations under the License. See accompanying LICENSE file.
<value>2m</value>
</property>
+ <property>
+ <name>yarn.nodemanager.resource-type.yarn.io/gpu</name>
+ <value>1</value>
+ </property>
+
</configuration>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml
index c84316a..ea8d2bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml
@@ -18,7 +18,7 @@ limitations under the License. See accompanying LICENSE file.
<property>
<name>yarn.resource-types</name>
- <value>resource1,resource2</value>
+ <value>resource1,resource2,yarn.io/gpu</value>
</property>
<property>
@@ -31,4 +31,9 @@ limitations under the License. See accompanying LICENSE file.
<value>m</value>
</property>
+ <property>
+ <name>yarn.resource-types.yarn.io/gpu.units</name>
+ <value></value>
+ </property>
+
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 07/09: YARN-9175. Null resources check in ResourceInfo for
branch-3.0
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 03303694d106cacf60f346200cf50467ecb62990
Author: Jonathan Hung <jh...@linkedin.com>
AuthorDate: Thu Jan 3 15:58:10 2019 -0500
YARN-9175. Null resources check in ResourceInfo for branch-3.0
---
.../hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
index e13980a..dd80d20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
@@ -62,7 +62,7 @@ public class ResourceInfo {
@Override
public String toString() {
- return resources.toString();
+ return getResource().toString();
}
public void setMemory(int memory) {
@@ -82,6 +82,9 @@ public class ResourceInfo {
}
public Resource getResource() {
+ if (resources == null) {
+ resources = Resource.newInstance(memory, vCores);
+ }
return Resource.newInstance(resources);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org