You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jb...@apache.org on 2020/10/06 18:37:33 UTC
[hadoop] branch branch-3.2 updated: YARN-10451. RM (v1) UI
NodesPage can NPE when yarn.io/gpu resource type is defined. Contributed by
Eric Payne
This is an automated email from the ASF dual-hosted git repository.
jbrennan pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new ecf9163 YARN-10451. RM (v1) UI NodesPage can NPE when yarn.io/gpu resource type is defined. Contributed by Eric Payne
ecf9163 is described below
commit ecf91638a8d1f546d90e92d076f32cb5681ce944
Author: Jim Brennan <jb...@apache.org>
AuthorDate: Tue Oct 6 18:36:51 2020 +0000
YARN-10451. RM (v1) UI NodesPage can NPE when yarn.io/gpu resource type is defined. Contributed by Eric Payne
---
.../server/resourcemanager/webapp/NodesPage.java | 3 ++-
.../resourcemanager/webapp/TestNodesPage.java | 26 ++++++++++++++++++++--
.../resourcemanager/webapp/TestRMWebApp.java | 23 +++++++++++++++++--
3 files changed, 47 insertions(+), 5 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
index 909bd1b..88516da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
@@ -175,7 +175,8 @@ class NodesPage extends RmView {
.get(ResourceInformation.GPU_URI);
long usedGPUs = 0;
long availableGPUs = 0;
- if (gpuIndex != null) {
+ if (gpuIndex != null && info.getUsedResource() != null
+ && info.getAvailableResource() != null) {
usedGPUs = info.getUsedResource().getResource()
.getResourceValue(ResourceInformation.GPU_URI);
availableGPUs = info.getAvailableResource().getResource()
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
index 1f6a8c0..51f1a40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
@@ -19,12 +19,16 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.Map;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.NodesPage.NodesBlock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
import org.junit.Before;
import org.junit.Test;
@@ -56,6 +60,10 @@ public class TestNodesPage {
@Before
public void setUp() throws Exception {
+ setUpInternal(false);
+ }
+
+ private void setUpInternal(final boolean useDRC) throws Exception {
final RMContext mockRMContext =
TestRMWebApp.mockRMContext(3, numberOfRacks, numberOfNodesPerRack,
8 * TestRMWebApp.GiB);
@@ -66,7 +74,7 @@ public class TestNodesPage {
public void configure(Binder binder) {
try {
binder.bind(ResourceManager.class).toInstance(
- TestRMWebApp.mockRm(mockRMContext));
+ TestRMWebApp.mockRm(mockRMContext, useDRC));
} catch (IOException e) {
throw new IllegalStateException(e);
}
@@ -101,7 +109,21 @@ public class TestNodesPage {
Mockito.verify(writer, Mockito.times(numberOfThInMetricsTable))
.print("<td");
}
-
+
+ @Test
+ public void testNodesBlockRenderForLostNodesWithGPUResources()
+ throws Exception {
+ Map<String, ResourceInformation> oldRtMap =
+ ResourceUtils.getResourceTypes();
+ TestResourceUtils.addNewTypesToResources(ResourceInformation.GPU_URI);
+ this.setUpInternal(true);
+ try {
+ this.testNodesBlockRenderForLostNodes();
+ } finally {
+ ResourceUtils.initializeResourcesFromResourceInformationMap(oldRtMap);
+ }
+ }
+
@Test
public void testNodesBlockRenderForNodeLabelFilterWithNonEmptyLabel() {
NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
index 93377be..c5586b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
@@ -210,8 +210,13 @@ public class TestRMWebApp {
}
public static ResourceManager mockRm(RMContext rmContext) throws IOException {
+ return mockRm(rmContext, false);
+ }
+
+ public static ResourceManager mockRm(RMContext rmContext,
+ boolean useDRC) throws IOException {
ResourceManager rm = mock(ResourceManager.class);
- ResourceScheduler rs = mockCapacityScheduler();
+ ResourceScheduler rs = mockCapacityScheduler(useDRC);
ApplicationACLsManager aclMgr = mockAppACLsManager();
ClientRMService clientRMService = mockClientRMService(rmContext);
when(rm.getResourceScheduler()).thenReturn(rs);
@@ -222,9 +227,14 @@ public class TestRMWebApp {
}
public static CapacityScheduler mockCapacityScheduler() throws IOException {
+ return mockCapacityScheduler(false);
+ }
+
+ public static CapacityScheduler mockCapacityScheduler(boolean useDRC)
+ throws IOException {
// stolen from TestCapacityScheduler
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
- setupQueueConfiguration(conf);
+ setupQueueConfiguration(conf, useDRC);
CapacityScheduler cs = new CapacityScheduler();
YarnConfiguration yarnConf = new YarnConfiguration();
@@ -276,6 +286,11 @@ public class TestRMWebApp {
static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
+ setupQueueConfiguration(conf, false);
+ }
+
+ static void setupQueueConfiguration(CapacitySchedulerConfiguration conf,
+ boolean useDRC) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
@@ -321,6 +336,10 @@ public class TestRMWebApp {
conf.setCapacity(C11, 15);
conf.setCapacity(C12, 45);
conf.setCapacity(C13, 40);
+ if (useDRC) {
+ conf.set("yarn.scheduler.capacity.resource-calculator",
+ "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator");
+ }
}
public static ResourceManager mockFifoRm(int apps, int racks, int nodes,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org