You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/02/25 01:50:18 UTC
[05/31] hadoop git commit: YARN-4709. NMWebServices produces
incorrect JSON for containers. Contributed by Varun Saxena.
YARN-4709. NMWebServices produces incorrect JSON for containers. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/140cb5d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/140cb5d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/140cb5d7
Branch: refs/heads/HDFS-1312
Commit: 140cb5d74565aad848b40a251ec02b597a3a20d4
Parents: 27b7775
Author: Varun Vasudev <vv...@apache.org>
Authored: Tue Feb 23 12:29:25 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Tue Feb 23 12:29:25 2016 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../yarn/webapp/WebServicesTestUtils.java | 28 ++++++++++++++
.../nodemanager/webapp/dao/ContainerInfo.java | 1 -
.../webapp/TestNMWebServicesContainers.java | 39 +++++++++++++++++---
4 files changed, 65 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/140cb5d7/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9b57666..058ea87 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1436,6 +1436,9 @@ Release 2.8.0 - UNRELEASED
YARN-4386. refreshNodesGracefully() should send recommission event to active
RMNodes only. (Kuhu Shukla via junping_du)
+ YARN-4709. NMWebServices produces incorrect JSON for containers.
+ (Varun Saxena via vvasudev)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/140cb5d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java
index aa211a6..0454ce6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.webapp;
import static org.junit.Assert.assertTrue;
+import java.util.ArrayList;
+import java.util.List;
+
import org.w3c.dom.Attr;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
@@ -47,6 +50,31 @@ public class WebServicesTestUtils {
return Float.parseFloat(val);
}
+ public static List<String> getXmlStrings(Element element, String name) {
+ NodeList id = element.getElementsByTagName(name);
+ List<String> strings = new ArrayList<>();
+ int len = id.getLength();
+ if (id.getLength() == 0) {
+ return strings;
+ }
+ for (int i = 0; i < len; i++) {
+ Element line = (Element) id.item(i);
+ if (line == null) {
+ continue;
+ }
+ Node first = line.getFirstChild();
+ if (first == null) {
+ continue;
+ }
+ String val = first.getNodeValue();
+ if (val == null) {
+ continue;
+ }
+ strings.add(val);
+ }
+ return strings;
+ }
+
public static String getXmlString(Element element, String name) {
NodeList id = element.getElementsByTagName(name);
Element line = (Element) id.item(0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/140cb5d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
index e462eba..cf022b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
@@ -53,7 +53,6 @@ public class ContainerInfo {
@XmlTransient
protected String exitStatus;
- @XmlElementWrapper
protected List<String> containerLogFiles;
public ContainerInfo() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/140cb5d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
index 0ed56d3..3c4a660 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
@@ -20,13 +20,16 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.List;
import javax.ws.rs.core.MediaType;
import javax.xml.parsers.DocumentBuilder;
@@ -48,6 +51,7 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -203,12 +207,27 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
app.getAppId(), 1);
Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
app.getUser(), app.getAppId(), 1);
+ ((MockContainer)container1).setState(ContainerState.RUNNING);
Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
app.getUser(), app.getAppId(), 2);
+ ((MockContainer)container2).setState(ContainerState.RUNNING);
nmContext.getContainers()
.put(container1.getContainerId(), container1);
nmContext.getContainers()
.put(container2.getContainerId(), container2);
+ File appDir = new File(testLogDir + "/" + app.getAppId().toString());
+ appDir.mkdir();
+ File container1Dir =
+ new File(appDir + "/" + container1.getContainerId().toString());
+ container1Dir.mkdir();
+ // Create log files for containers.
+ new File(container1Dir + "/" + "syslog").createNewFile();
+ new File(container1Dir + "/" + "stdout").createNewFile();
+ File container2Dir =
+ new File(appDir + "/" + container2.getContainerId().toString());
+ container2Dir.mkdir();
+ new File(container2Dir + "/" + "syslog").createNewFile();
+ new File(container2Dir + "/" + "stdout").createNewFile();
app.getContainers().put(container1.getContainerId(), container1);
app.getContainers().put(container2.getContainerId(), container2);
@@ -475,9 +494,13 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"),
WebServicesTestUtils.getXmlString(element, "containerLogsLink"));
// verify that the container log files element exists
- assertTrue("containerLogFiles missing",
- WebServicesTestUtils.getXmlString(element, "containerLogFiles")
- != null);
+ List<String> containerLogFiles =
+ WebServicesTestUtils.getXmlStrings(element, "containerLogFiles");
+ assertFalse("containerLogFiles missing",containerLogFiles.isEmpty());
+ assertEquals(2, containerLogFiles.size());
+ assertTrue("syslog and stdout expected",
+ containerLogFiles.contains("syslog") &&
+ containerLogFiles.contains("stdout"));
}
}
@@ -492,8 +515,14 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
info.getInt("totalVCoresNeeded"),
info.getString("containerLogsLink"));
// verify that the container log files element exists
- assertTrue("containerLogFiles missing",
- info.getJSONArray("containerLogFiles") != null);
+ JSONArray containerLogFilesArr = info.getJSONArray("containerLogFiles");
+ assertTrue("containerLogFiles missing", containerLogFilesArr != null);
+ assertEquals(2, containerLogFilesArr.length());
+ for (int i = 0; i < 2; i++) {
+ assertTrue("syslog and stdout expected",
+ containerLogFilesArr.get(i).equals("syslog") ||
+ containerLogFilesArr.get(i).equals("stdout"));
+ }
}
public void verifyNodeContainerInfoGeneric(Container cont, String id,