[hadoop] branch branch-3.2 updated: YARN-9213. RM Web UI v1 does not show custom resource allocations for containers page. Contributed by Szilard Nemeth.

[su...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
     new f282f9c  YARN-9213. RM Web UI v1 does not show custom resource allocations for containers page. Contributed by Szilard Nemeth.
f282f9c is described below

commit f282f9c362dc5a2b7a86b7e2f78d53835c2a6188
Author: Sunil G <su...@apache.org>
AuthorDate: Mon Feb 25 11:37:42 2019 +0530

    YARN-9213. RM Web UI v1 does not show custom resource allocations for containers page. Contributed by Szilard Nemeth.
---
 .../hadoop/yarn/server/webapp/ContainerBlock.java  |  66 +++++++++--
 .../yarn/server/webapp/dao/ContainerInfo.java      |   4 +
 .../yarn/server/webapp/ContainerBlockTest.java     | 126 +++++++++++++++++++++
 3 files changed, 185 insertions(+), 11 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
index fd48dc5..2bf643f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
@@ -17,28 +17,30 @@
  */
 package org.apache.hadoop.yarn.server.webapp;
 
-import static org.apache.hadoop.yarn.util.StringHelper.join;
-import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID;
-
-import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.inject.Inject;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
 import org.apache.hadoop.yarn.util.Times;
 import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
 import org.apache.hadoop.yarn.webapp.view.InfoBlock;
-
-import com.google.inject.Inject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Map;
+
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID;
+
 public class ContainerBlock extends HtmlBlock {
 
   private static final Logger LOG =
@@ -117,9 +119,7 @@ public class ContainerBlock extends HtmlBlock {
         StringUtils.formatTime(Times.elapsed(container.getStartedTime(),
           container.getFinishedTime())))
       .__(
-        "Resource:",
-        container.getAllocatedMB() + " Memory, "
-            + container.getAllocatedVCores() + " VCores")
+        "Resource:", getResources(container))
       .__("Logs:", container.getLogUrl() == null ? "#" : container.getLogUrl(),
           container.getLogUrl() == null ? "N/A" : "Logs")
       .__("Diagnostics:", container.getDiagnosticsInfo() == null ?
@@ -128,6 +128,50 @@ public class ContainerBlock extends HtmlBlock {
     html.__(InfoBlock.class);
   }
 
+  /**
+   * Creates a string representation of allocated resources to a container.
+   * Memory, followed with VCores are always the first two resources of
+   * the resulted string, followed with any custom resources, if any is present.
+   */
+  @VisibleForTesting
+  String getResources(ContainerInfo container) {
+    Map<String, Long> allocatedResources = container.getAllocatedResources();
+
+    StringBuilder sb = new StringBuilder();
+    sb.append(getResourceAsString(ResourceInformation.MEMORY_URI,
+        allocatedResources.get(ResourceInformation.MEMORY_URI))).append(", ");
+    sb.append(getResourceAsString(ResourceInformation.VCORES_URI,
+        allocatedResources.get(ResourceInformation.VCORES_URI)));
+
+    if (container.hasCustomResources()) {
+      container.getAllocatedResources().forEach((key, value) -> {
+        if (!key.equals(ResourceInformation.MEMORY_URI) &&
+            !key.equals(ResourceInformation.VCORES_URI)) {
+          sb.append(", ");
+          sb.append(getResourceAsString(key, value));
+        }
+      });
+    }
+
+    return sb.toString();
+  }
+
+  private String getResourceAsString(String resourceName, long value) {
+    final String translatedResourceName;
+    switch (resourceName) {
+    case ResourceInformation.MEMORY_URI:
+      translatedResourceName = "Memory";
+      break;
+    case ResourceInformation.VCORES_URI:
+      translatedResourceName = "VCores";
+      break;
+    default:
+      translatedResourceName = resourceName;
+      break;
+    }
+    return String.valueOf(value) + " " + translatedResourceName;
+  }
+
   protected ContainerReport getContainerReport(
       final GetContainerReportRequest request)
       throws YarnException, IOException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
index 26a822c..d68ce46 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
@@ -159,4 +159,8 @@ public class ContainerInfo {
   public Map<String, Long> getAllocatedResources() {
     return Collections.unmodifiableMap(allocatedResources);
   }
+
+  public boolean hasCustomResources() {
+    return allocatedResources.size() > 2;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/ContainerBlockTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/ContainerBlockTest.java
new file mode 100644
index 0000000..3c526fe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/webapp/ContainerBlockTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.webapp;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
+import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper;
+import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.webapp.View;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Tests for ContainerBlock.
+ */
+public class ContainerBlockTest {
+  private static class CustomResourceTypesConfigurationProvider
+      extends LocalConfigurationProvider {
+
+    private static String resourceName;
+
+    @Override
+    public InputStream getConfigurationInputStream(Configuration bootstrapConf,
+        String name) throws YarnException, IOException {
+      if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) {
+        return new ByteArrayInputStream(
+            ("<configuration>\n" +
+            " <property>\n" +
+            "   <name>yarn.resource-types</name>\n" +
+            "   <value>" + resourceName + "</value>\n" +
+            " </property>\n" +
+            " <property>\n" +
+            "   <name>yarn.resource-types.a-custom-resource.units</name>\n" +
+            "   <value>G</value>\n" +
+            " </property>\n" +
+            "</configuration>\n").getBytes());
+      } else {
+        return super.getConfigurationInputStream(bootstrapConf, name);
+      }
+    }
+
+    public static String getResourceName() {
+      return resourceName;
+    }
+  }
+
+  private void initResourceTypes(String resourceName) {
+    CustomResourceTypesConfigurationProvider.resourceName = resourceName;
+
+    Configuration configuration = new Configuration();
+    configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+        CustomResourceTypesConfigurationProvider.class.getName());
+    ResourceUtils.resetResourceTypes(configuration);
+  }
+
+  private ContainerReport createContainerReport() {
+    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
+        applicationId, 1);
+    ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
+    ContainerReport container = ContainerReport.newInstance(containerId, null,
+        NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
+        "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+        "http://" + NodeId.newInstance("host", 2345).toString());
+
+    return container;
+  }
+  @Test
+  public void testRenderResourcesString() {
+    initResourceTypes(ResourceInformation.GPU_URI);
+
+    Resource resource = ResourceTypesTestHelper.newResource(
+        DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        ImmutableMap.<String, String>builder()
+            .put(ResourceInformation.GPU_URI, "5").build());
+
+    ContainerBlock block = new ContainerBlock(
+        mock(ApplicationBaseProtocol.class),
+        mock(View.ViewContext.class));
+
+    ContainerReport containerReport = createContainerReport();
+    containerReport.setAllocatedResource(resource);
+    ContainerInfo containerInfo = new ContainerInfo(containerReport);
+    String resources = block.getResources(containerInfo);
+
+    Assert.assertEquals("8192 Memory, 4 VCores, 5 yarn.io/gpu", resources);
+  }
+
+}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org