You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2019/02/08 20:09:14 UTC
[hadoop] branch YARN-8200 updated (2252b19 -> d9616d6)
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a change to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 2252b19 YARN-9289. Backport YARN-7330 for GPU in UI to branch-2
new 2158fbd YARN-7573. Gpu Information page could be empty for nodes without GPU. (Sunil G via wangda)
new d2274c6 YARN-7594. TestNMWebServices#testGetNMResourceInfo fails on trunk. Contributed by Gergely Novák.
new d9616d6 YARN-7223. Document GPU isolation feature. Contributed by Wangda Tan.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../server/nodemanager/webapp/NMWebServices.java | 4 +-
.../nodemanager/webapp/TestNMWebServices.java | 12 +-
.../src/site/markdown/UsingGpus.md | 230 +++++++++++++++++++++
.../src/main/webapp/app/models/yarn-rm-node.js | 6 +-
.../app/templates/components/node-menu-panel.hbs | 2 +-
.../main/webapp/app/templates/yarn-node-apps.hbs | 2 +-
.../webapp/app/templates/yarn-node-containers.hbs | 2 +-
.../webapp/app/templates/yarn-node/yarn-nm-gpu.hbs | 4 +
8 files changed, 246 insertions(+), 16 deletions(-)
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingGpus.md
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/03: YARN-7573. Gpu Information page could be empty for
nodes without GPU. (Sunil G via wangda)
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 2158fbd9517e9cae3ffac172c4ff572b808fd55e
Author: Wangda Tan <wa...@apache.org>
AuthorDate: Wed Nov 29 17:43:37 2017 -0800
YARN-7573. Gpu Information page could be empty for nodes without GPU. (Sunil G via wangda)
Change-Id: I7f614e5a589a09ce4e4286c84b706e05c29abd14
---
.../apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java | 4 +---
.../hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js | 6 ++++--
.../src/main/webapp/app/templates/components/node-menu-panel.hbs | 2 +-
.../hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-apps.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-node-containers.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs | 4 ++++
6 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index 7476d75..7702004 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -510,9 +510,7 @@ public class NMWebServices {
}
}
- throw new YarnException(
- "Could not get detailed resource information for given resource-name="
- + resourceName);
+ return new NMResourceInfo();
}
private long parseLongParam(String bytes) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
index b1b1518..aa5efbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
@@ -97,7 +97,9 @@ export default DS.Model.extend({
var used = 0;
var ri;
- var resourceInformations = this.get("usedResource").resourcesInformations;
+ const usedResource = this.get("usedResource");
+ const availableResource = this.get("availableResource");
+ var resourceInformations = usedResource ? usedResource.resourcesInformations : [];
for (var i = 0; i < resourceInformations.length; i++) {
ri = resourceInformations[i];
if (ri.name === "yarn.io/gpu") {
@@ -106,7 +108,7 @@ export default DS.Model.extend({
}
var available = 0;
- resourceInformations = this.get("availableResource").resourcesInformations;
+ resourceInformations = availableResource ? availableResource.resourcesInformations : [];
for (i = 0; i < resourceInformations.length; i++) {
ri = resourceInformations[i];
if (ri.name === "yarn.io/gpu") {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs
index fffae30..966e408 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs
@@ -36,7 +36,7 @@
{{#link-to 'yarn-node-containers' nodeId nodeAddr}}List of Containers
{{/link-to}}
{{/link-to}}
- {{#if nmGpuInfo}}
+ {{#if (and nmGpuInfo nmGpuInfo.info.totalGpuDevices)}}
{{#link-to 'yarn-node.yarn-nm-gpu' tagName="li"}}
{{#link-to 'yarn-node.yarn-nm-gpu' nodeId nodeAddr }}GPU Information
{{/link-to}}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-apps.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-apps.hbs
index 52f0c86..919e54d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-apps.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-apps.hbs
@@ -20,7 +20,7 @@
<div class="col-md-12 container-fluid">
<div class="row">
- {{node-menu-panel path="yarn-node-apps" nodeAddr=model.nodeInfo.addr nodeId=model.nodeInfo.id}}
+ {{node-menu-panel path="yarn-node-apps" nodeAddr=model.nodeInfo.addr nodeId=model.nodeInfo.id nmGpuInfo=model.nmGpuInfo}}
{{#if model.apps}}
<div class="col-md-10 container-fluid">
<table id="node-apps-table" class="display table table-striped table-bordered" cellspacing="0" width="100%">
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-containers.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-containers.hbs
index f520c46..1f31272 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-containers.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node-containers.hbs
@@ -20,7 +20,7 @@
<div class="col-md-12 container-fluid">
<div class="row">
- {{node-menu-panel path="yarn-node-containers" nodeAddr=model.nodeInfo.addr nodeId=model.nodeInfo.id}}
+ {{node-menu-panel path="yarn-node-containers" nodeAddr=model.nodeInfo.addr nodeId=model.nodeInfo.id nmGpuInfo=model.nmGpuInfo}}
{{#if model.containers}}
<div class="col-md-10 container-fluid">
<table id="node-containers-table" class="display table table-striped table-bordered" cellspacing="0" width="100%">
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs
index 55840ad..0464cc8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs
@@ -23,6 +23,7 @@
{{node-menu-panel path="yarn-node" nodeId=model.rmNode.id
nodeAddr=model.node.id nmGpuInfo=model.nmGpuInfo}}
+ {{#if model.nmGpuInfo.info.totalGpuDevices}}
<div class="col-md-10 container-fluid">
<div class="panel panel-default">
@@ -49,5 +50,8 @@
{{yarn-nm-gpu-info gpu=gpu}}
{{/each}}
</div>
+ {{else}}
+ <h4 align = "center">No GPUs are found on this node.</h4>
+ {{/if}}
</div>
</div>
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 03/03: YARN-7223. Document GPU isolation feature.
Contributed by Wangda Tan.
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit d9616d6ab0d629f860d9ff4768e8415aa79094b0
Author: Sunil G <su...@apache.org>
AuthorDate: Wed Feb 21 14:16:45 2018 +0530
YARN-7223. Document GPU isolation feature. Contributed by Wangda Tan.
---
.../src/site/markdown/UsingGpus.md | 230 +++++++++++++++++++++
1 file changed, 230 insertions(+)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingGpus.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingGpus.md
new file mode 100644
index 0000000..f6000e7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingGpus.md
@@ -0,0 +1,230 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+
+# Using GPU On YARN
+# Prerequisites
+
+- As of now, only Nvidia GPUs are supported by YARN
+- YARN node managers have to be pre-installed with Nvidia drivers.
+- When Docker is used as container runtime context, nvidia-docker 1.0 needs to be installed (Current supported version in YARN for nvidia-docker).
+
+# Configs
+
+## GPU scheduling
+
+In `resource-types.xml`
+
+Add following properties
+
+```
+<configuration>
+ <property>
+ <name>yarn.resource-types</name>
+ <value>yarn.io/gpu</value>
+ </property>
+</configuration>
+```
+
+In `yarn-site.xml`
+
+`DominantResourceCalculator` MUST be configured to enable GPU scheduling/isolation.
+
+For `Capacity Scheduler`, use following property to configure `DominantResourceCalculator` (In `capacity-scheduler.xml`):
+
+| Property | Default value |
+| --- | --- |
+| yarn.scheduler.capacity.resource-calculator | org.apache.hadoop.yarn.util.resource.DominantResourceCalculator |
+
+
+## GPU Isolation
+
+### In `yarn-site.xml`
+
+```
+ <property>
+ <name>yarn.nodemanager.resource-plugins</name>
+ <value>yarn.io/gpu</value>
+ </property>
+```
+
+This is to enable GPU isolation module on NodeManager side.
+
+By default, YARN will automatically detect and config GPUs when above config is set. Following configs need to be set in `yarn-site.xml` only if admin has specialized requirements.
+
+**1) Allowed GPU Devices**
+
+| Property | Default value |
+| --- | --- |
+| yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices | auto |
+
+ Specify GPU devices which can be managed by YARN NodeManager (split by comma).
+ Number of GPU devices will be reported to RM to make scheduling decisions.
+ Set to auto (default) let YARN automatically discover GPU resource from
+ system.
+
+ Manually specify GPU devices if auto detect GPU device failed or admin
+ only want subset of GPU devices managed by YARN. GPU device is identified
+ by their minor device number and index. A common approach to get minor
+ device number of GPUs is using `nvidia-smi -q` and search `Minor Number`
+ output.
+
+ When minor numbers are specified manually, admin needs to include indice of GPUs
+ as well, format is `index:minor_number[,index:minor_number...]`. An example
+ of manual specification is `0:0,1:1,2:2,3:4"`to allow YARN NodeManager to
+ manage GPU devices with indices `0/1/2/3` and minor number `0/1/2/4`.
+ numbers .
+
+**2) Executable to discover GPUs**
+
+| Property | value |
+| --- | --- |
+| yarn.nodemanager.resource-plugins.gpu.path-to-discovery-executables | /absolute/path/to/nvidia-smi |
+
+When `yarn.nodemanager.resource.gpu.allowed-gpu-devices=auto` specified,
+YARN NodeManager needs to run GPU discovery binary (now only support
+`nvidia-smi`) to get GPU-related information.
+When value is empty (default), YARN NodeManager will try to locate
+discovery executable itself.
+An example of the config value is: `/usr/local/bin/nvidia-smi`
+
+**3) Docker Plugin Related Configs**
+
+Following configs can be customized when user needs to run GPU applications inside Docker container. They're not required if admin follows default installation/configuration of `nvidia-docker`.
+
+| Property | Default value |
+| --- | --- |
+| yarn.nodemanager.resource-plugins.gpu.docker-plugin | nvidia-docker-v1 |
+
+Specify docker command plugin for GPU. By default uses Nvidia docker V1.0.
+
+| Property | Default value |
+| --- | --- |
+| yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint | http://localhost:3476/v1.0/docker/cli |
+
+Specify end point of `nvidia-docker-plugin`. Please find documentation: https://github.com/NVIDIA/nvidia-docker/wiki For more details.
+
+**4) CGroups mount**
+
+GPU isolation uses CGroup [devices controller](https://www.kernel.org/doc/Documentation/cgroup-v1/devices.txt) to do per-GPU device isolation. Following configs should be added to `yarn-site.xml` to automatically mount CGroup sub devices, otherwise admin has to manually create devices subfolder in order to use this feature.
+
+| Property | Default value |
+| --- | --- |
+| yarn.nodemanager.linux-container-executor.cgroups.mount | true |
+
+
+### In `container-executor.cfg`
+
+In general, following config needs to be added to `container-executor.cfg`
+
+```
+[gpu]
+module.enabled=true
+```
+
+When user needs to run GPU applications under non-Docker environment:
+
+```
+[cgroups]
+# This should be same as yarn.nodemanager.linux-container-executor.cgroups.mount-path inside yarn-site.xml
+root=/sys/fs/cgroup
+# This should be same as yarn.nodemanager.linux-container-executor.cgroups.hierarchy inside yarn-site.xml
+yarn-hierarchy=yarn
+```
+
+When user needs to run GPU applications under Docker environment:
+
+**1) Add GPU related devices to docker section:**
+
+Values separated by comma, you can get this by running `ls /dev/nvidia*`
+
+```
+[docker]
+docker.allowed.devices=/dev/nvidiactl,/dev/nvidia-uvm,/dev/nvidia-uvm-tools,/dev/nvidia1,/dev/nvidia0
+```
+
+**2) Add `nvidia-docker` to volume-driver whitelist.**
+
+```
+[docker]
+...
+docker.allowed.volume-drivers
+```
+
+**3) Add `nvidia_driver_<version>` to readonly mounts whitelist.**
+
+```
+[docker]
+...
+docker.allowed.ro-mounts=nvidia_driver_375.66
+```
+
+# Use it
+
+## Distributed-shell + GPU
+
+Distributed shell currently support specify additional resource types other than memory and vcores.
+
+### Distributed-shell + GPU without Docker
+
+Run distributed shell without using docker container (Asks 2 tasks, each task has 3GB memory, 1 vcore, 2 GPU device resource):
+
+```
+yarn jar <path/to/hadoop-yarn-applications-distributedshell.jar> \
+ -jar <path/to/hadoop-yarn-applications-distributedshell.jar> \
+ -shell_command /usr/local/nvidia/bin/nvidia-smi \
+ -container_resources memory-mb=3072,vcores=1,yarn.io/gpu=2 \
+ -num_containers 2
+```
+
+You should be able to see output like
+
+```
+Tue Dec 5 22:21:47 2017
++-----------------------------------------------------------------------------+
+| NVIDIA-SMI 375.66 Driver Version: 375.66 |
+|-------------------------------+----------------------+----------------------+
+| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
+| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
+|===============================+======================+======================|
+| 0 Tesla P100-PCIE... Off | 0000:04:00.0 Off | 0 |
+| N/A 30C P0 24W / 250W | 0MiB / 12193MiB | 0% Default |
++-------------------------------+----------------------+----------------------+
+| 1 Tesla P100-PCIE... Off | 0000:82:00.0 Off | 0 |
+| N/A 34C P0 25W / 250W | 0MiB / 12193MiB | 0% Default |
++-------------------------------+----------------------+----------------------+
+
++-----------------------------------------------------------------------------+
+| Processes: GPU Memory |
+| GPU PID Type Process name Usage |
+|=============================================================================|
+| No running processes found |
++-----------------------------------------------------------------------------+
+```
+
+For launched container task.
+
+### Distributed-shell + GPU with Docker
+
+You can also run distributed shell with Docker container. `YARN_CONTAINER_RUNTIME_TYPE`/`YARN_CONTAINER_RUNTIME_DOCKER_IMAGE` must be specified to use docker container.
+
+```
+yarn jar <path/to/hadoop-yarn-applications-distributedshell.jar> \
+ -jar <path/to/hadoop-yarn-applications-distributedshell.jar> \
+ -shell_env YARN_CONTAINER_RUNTIME_TYPE=docker \
+ -shell_env YARN_CONTAINER_RUNTIME_DOCKER_IMAGE=<docker-image-name> \
+ -shell_command nvidia-smi \
+ -container_resources memory-mb=3072,vcores=1,yarn.io/gpu=2 \
+ -num_containers 2
+```
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/03: YARN-7594. TestNMWebServices#testGetNMResourceInfo fails on trunk. Contributed by Gergely Novák.
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit d2274c64126fa5ec06d4797e989315071aa71469
Author: Sunil G <su...@apache.org>
AuthorDate: Mon Dec 4 10:45:07 2017 +0530
YARN-7594. TestNMWebServices#testGetNMResourceInfo fails on trunk. Contributed by Gergely Novák.
---
.../yarn/server/nodemanager/webapp/TestNMWebServices.java | 12 ++++--------
1 file changed, 4 insertions(+), 8 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
index 72071da..2f1577f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
@@ -456,17 +456,17 @@ public class TestNMWebServices extends JerseyTestBase {
ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON, response.getType().toString());
- // Access resource-2 should fail (null NMResourceInfo returned).
+ // Access resource-2 should fail (empty NMResourceInfo returned).
JSONObject json = response.getEntity(JSONObject.class);
- assertIncludesException(json);
+ assertEquals(0, json.length());
- // Access resource-3 should fail (unkown plugin)
+ // Access resource-3 should fail (unknown plugin)
response = r.path("ws").path("v1").path("node").path(
"resources").path("resource-3").accept(MediaType.APPLICATION_JSON).get(
ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON, response.getType().toString());
json = response.getEntity(JSONObject.class);
- assertIncludesException(json);
+ assertEquals(0, json.length());
// Access resource-1 should success
response = r.path("ws").path("v1").path("node").path(
@@ -533,10 +533,6 @@ public class TestNMWebServices extends JerseyTestBase {
assertEquals(2, json.getJSONArray("assignedGpuDevices").length());
}
- private void assertIncludesException(JSONObject json) {
- assertTrue(json.has("RemoteException"));
- }
-
private void testContainerLogs(WebResource r, ContainerId containerId)
throws IOException {
final String containerIdStr = containerId.toString();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org