You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2015/09/23 10:10:01 UTC
[03/50] [abbrv] hadoop git commit: YARN-4149. yarn logs -am should
provide an option to fetch all the log files. Contributed by Varun Vasudev
YARN-4149. yarn logs -am should provide an option to fetch all the log
files. Contributed by Varun Vasudev
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34ef1a09
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34ef1a09
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34ef1a09
Branch: refs/heads/HDFS-7966
Commit: 34ef1a092bcab369bb845481efffb8c47adef53a
Parents: 8c1cdb1
Author: Xuan <xg...@apache.org>
Authored: Tue Sep 15 14:36:30 2015 -0700
Committer: Xuan <xg...@apache.org>
Committed: Tue Sep 15 14:36:30 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 104 +++++++++++++++++--
.../hadoop/yarn/client/cli/TestLogsCLI.java | 3 +-
.../nodemanager/webapp/NMWebServices.java | 12 ++-
.../nodemanager/webapp/dao/ContainerInfo.java | 46 ++++++--
.../webapp/TestNMWebServicesContainers.java | 13 ++-
6 files changed, 157 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ef1a09/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b2fb780..b044078 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -439,6 +439,9 @@ Release 2.8.0 - UNRELEASED
YARN-3717. Expose app/am/queue's node-label-expression to RM web UI /
CLI / REST-API. (Naganarasimha G R via wangda)
+ YARN-4149. yarn logs -am should provide an option to fetch all the log files
+ (Varun Vasudev via xgong)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ef1a09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 25481f8..edee8ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -19,12 +19,15 @@
package org.apache.hadoop.yarn.client.cli;
import java.io.IOException;
+import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -61,6 +64,9 @@ import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
@Public
@Evolving
@@ -105,7 +111,8 @@ public class LogsCLI extends Configured implements Tool {
opts.addOption(amOption);
Option logFileOpt = new Option(CONTAINER_LOG_FILES, true,
"Work with -am/-containerId and specify comma-separated value "
- + "to get specified Container log files");
+ + "to get specified container log files. Use \"ALL\" to fetch all the "
+ + "log files for the container.");
logFileOpt.setValueSeparator(',');
logFileOpt.setArgs(Option.UNLIMITED_VALUES);
logFileOpt.setArgName("Log File Name");
@@ -248,8 +255,8 @@ public class LogsCLI extends Configured implements Tool {
logFiles, logCliHelper, appOwner, true);
} else {
System.out
- .println("Can not get AMContainers logs for the application:"
- + appId);
+ .println(
+ "Can not get AMContainers logs for the application:" + appId);
System.out.println("This application:" + appId + " is finished."
+ " Please enable the application history service. Or Using "
+ "yarn logs -applicationId <appId> -containerId <containerId> "
@@ -264,9 +271,18 @@ public class LogsCLI extends Configured implements Tool {
// if we provide the node address and the application is in the final
// state, we could directly get logs from HDFS.
if (nodeAddress != null && isApplicationFinished(appState)) {
+ // if user specified "ALL" as the logFiles param, pass null
+ // to logCliHelper so that it fetches all the logs
+ List<String> logs;
+ if (logFiles == null) {
+ logs = null;
+ } else if (fetchAllLogFiles(logFiles)) {
+ logs = null;
+ } else {
+ logs = Arrays.asList(logFiles);
+ }
return logCliHelper.dumpAContainersLogsForALogType(appIdStr,
- containerIdStr, nodeAddress, appOwner, logFiles == null ? null
- : Arrays.asList(logFiles));
+ containerIdStr, nodeAddress, appOwner, logs);
}
try {
// If the nodeAddress is not provided, we will try to get
@@ -288,10 +304,14 @@ public class LogsCLI extends Configured implements Tool {
containerIdStr, nodeHttpAddress, nodeId, logFiles, logCliHelper,
appOwner);
} else {
+ String [] requestedLogFiles = logFiles;
+ if(fetchAllLogFiles(logFiles)) {
+ requestedLogFiles = null;
+ }
// If the application is in the final state, we will directly
// get the container logs from HDFS.
printContainerLogsForFinishedApplication(appIdStr, containerIdStr,
- nodeId, logFiles, logCliHelper, appOwner);
+ nodeId, requestedLogFiles, logCliHelper, appOwner);
}
return resultCode;
} catch (IOException | YarnException ex) {
@@ -401,15 +421,69 @@ public class LogsCLI extends Configured implements Tool {
return amContainersList;
}
+ private boolean fetchAllLogFiles(String[] logFiles) {
+ if(logFiles != null) {
+ List<String> logs = Arrays.asList(logFiles);
+ if(logs.contains("ALL")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private String[] getContainerLogFiles(Configuration conf,
+ String containerIdStr, String nodeHttpAddress) throws IOException {
+ List<String> logFiles = new ArrayList<>();
+ Client webServiceClient = Client.create();
+ try {
+ WebResource webResource = webServiceClient
+ .resource(WebAppUtils.getHttpSchemePrefix(conf) + nodeHttpAddress);
+ ClientResponse response =
+ webResource.path("ws").path("v1").path("node").path("containers")
+ .path(containerIdStr).accept(MediaType.APPLICATION_XML)
+ .get(ClientResponse.class);
+ if (response.getClientResponseStatus().equals(ClientResponse.Status.OK)) {
+ try {
+ String xml = response.getEntity(String.class);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList elements = dom.getElementsByTagName("containerLogFiles");
+ for (int i = 0; i < elements.getLength(); i++) {
+ logFiles.add(elements.item(i).getTextContent());
+ }
+ } catch (Exception e) {
+ System.out.println("Unable to parse xml from webservice. Error:");
+ System.out.println(e.getMessage());
+ throw new IOException(e);
+ }
+ }
+
+ } catch (ClientHandlerException | UniformInterfaceException ex) {
+ System.out.println("Unable to fetch log files list");
+ throw new IOException(ex);
+ }
+ return logFiles.toArray(new String[0]);
+ }
+
private void printContainerLogsFromRunningApplication(Configuration conf,
String appId, String containerIdStr, String nodeHttpAddress,
String nodeId, String[] logFiles, LogCLIHelpers logCliHelper,
String appOwner) throws IOException {
+ String [] requestedLogFiles = logFiles;
+ // fetch all the log files for the container
+ if (fetchAllLogFiles(logFiles)) {
+ requestedLogFiles =
+ getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress);
+ }
Client webServiceClient = Client.create();
String containerString = "\n\nContainer: " + containerIdStr;
System.out.println(containerString);
System.out.println(StringUtils.repeat("=", containerString.length()));
- for (String logFile : logFiles) {
+
+ for (String logFile : requestedLogFiles) {
System.out.println("LogType:" + logFile);
System.out.println("Log Upload Time:"
+ Times.format(System.currentTimeMillis()));
@@ -432,7 +506,7 @@ public class LogsCLI extends Configured implements Tool {
}
// for the case, we have already uploaded partial logs in HDFS
logCliHelper.dumpAContainersLogsForALogType(appId, containerIdStr, nodeId,
- appOwner, Arrays.asList(logFiles));
+ appOwner, Arrays.asList(requestedLogFiles));
}
private void printContainerLogsForFinishedApplication(String appId,
@@ -551,15 +625,25 @@ public class LogsCLI extends Configured implements Tool {
}
}
if (nodeId != null && !nodeId.isEmpty()) {
+ String [] requestedLogFilesList = null;
+ if(!fetchAllLogFiles(logFiles)) {
+ requestedLogFilesList = logFiles;
+ }
printContainerLogsForFinishedApplication(appId, containerId, nodeId,
- logFiles, logCliHelper, appOwner);
+ requestedLogFilesList, logCliHelper, appOwner);
}
}
} else {
if (nodeHttpAddress != null && containerId != null
&& !nodeHttpAddress.isEmpty() && !containerId.isEmpty()) {
+ String [] requestedLogFiles = logFiles;
+ // fetch all the log files for the AM
+ if (fetchAllLogFiles(logFiles)) {
+ requestedLogFiles =
+ getContainerLogFiles(getConf(), containerId, nodeHttpAddress);
+ }
printContainerLogsFromRunningApplication(conf, appId, containerId,
- nodeHttpAddress, nodeId, logFiles, logCliHelper, appOwner);
+ nodeHttpAddress, nodeId, requestedLogFiles, logCliHelper, appOwner);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ef1a09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index a353811..aec7cae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -168,7 +168,8 @@ public class TestLogsCLI {
pw.println(" -help Displays help for all commands.");
pw.println(" -logFiles <Log File Name> Work with -am/-containerId and specify");
pw.println(" comma-separated value to get specified");
- pw.println(" Container log files");
+ pw.println(" container log files. Use \"ALL\" to fetch");
+ pw.println(" all the log files for the container.");
pw.println(" -nodeAddress <Node Address> NodeAddress in the format nodename:port");
pw.close();
String appReportStr = baos.toString("UTF-8");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ef1a09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index 16f2c68..de6d219 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -129,7 +129,7 @@ public class NMWebServices {
String msg = "Error: You must specify a non-empty string for the user";
throw new BadRequestException(msg);
}
- if (!appInfo.getUser().toString().equals(userQuery)) {
+ if (!appInfo.getUser().equals(userQuery)) {
continue;
}
}
@@ -158,7 +158,8 @@ public class NMWebServices {
@GET
@Path("/containers")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public ContainersInfo getNodeContainers() {
+ public ContainersInfo getNodeContainers(@javax.ws.rs.core.Context
+ HttpServletRequest hsr) {
init();
ContainersInfo allContainers = new ContainersInfo();
for (Entry<ContainerId, Container> entry : this.nmContext.getContainers()
@@ -168,7 +169,7 @@ public class NMWebServices {
continue;
}
ContainerInfo info = new ContainerInfo(this.nmContext, entry.getValue(),
- uriInfo.getBaseUri().toString(), webapp.name());
+ uriInfo.getBaseUri().toString(), webapp.name(), hsr.getRemoteUser());
allContainers.add(info);
}
return allContainers;
@@ -177,7 +178,8 @@ public class NMWebServices {
@GET
@Path("/containers/{containerid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public ContainerInfo getNodeContainer(@PathParam("containerid") String id) {
+ public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context
+ HttpServletRequest hsr, @PathParam("containerid") String id) {
ContainerId containerId = null;
init();
try {
@@ -191,7 +193,7 @@ public class NMWebServices {
throw new NotFoundException("container with id, " + id + ", not found");
}
return new ContainerInfo(this.nmContext, container, uriInfo.getBaseUri()
- .toString(), webapp.name());
+ .toString(), webapp.name(), hsr.getRemoteUser());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ef1a09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
index 78982de..e462eba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
@@ -21,16 +21,19 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp.dao;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-import javax.xml.bind.annotation.XmlTransient;
+import javax.xml.bind.annotation.*;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsUtils;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
@XmlRootElement(name = "container")
@XmlAccessorType(XmlAccessType.FIELD)
@@ -50,15 +53,18 @@ public class ContainerInfo {
@XmlTransient
protected String exitStatus;
+ @XmlElementWrapper
+ protected List<String> containerLogFiles;
+
public ContainerInfo() {
} // JAXB needs this
public ContainerInfo(final Context nmContext, final Container container) {
- this(nmContext, container, "", "");
+ this(nmContext, container, "", "", "");
}
public ContainerInfo(final Context nmContext, final Container container,
- String requestUri, String pathPrefix) {
+ String requestUri, String pathPrefix, String remoteUser) {
this.id = container.getContainerId().toString();
this.nodeId = nmContext.getNodeId().toString();
@@ -90,6 +96,8 @@ public class ContainerInfo {
}
this.containerLogsLink = join(requestUri, pathPrefix,
this.containerLogsShortLink);
+ this.containerLogFiles =
+ getContainerLogFiles(container.getContainerId(), remoteUser, nmContext);
}
public String getId() {
@@ -136,4 +144,30 @@ public class ContainerInfo {
return this.totalVCoresNeeded;
}
+ public List<String> getContainerLogFiles() {
+ return this.containerLogFiles;
+ }
+
+ private List<String> getContainerLogFiles(ContainerId id, String remoteUser,
+ Context nmContext) {
+ List<String> logFiles = new ArrayList<>();
+ try {
+ List<File> logDirs =
+ ContainerLogsUtils.getContainerLogDirs(id, remoteUser, nmContext);
+ for (File containerLogsDir : logDirs) {
+ File[] logs = containerLogsDir.listFiles();
+ if (logs != null) {
+ for (File log : logs) {
+ if (log.isFile()) {
+ logFiles.add(log.getName());
+ }
+ }
+ }
+ }
+ } catch (Exception ye) {
+ return logFiles;
+ }
+ return logFiles;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34ef1a09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
index 3bfd440..0ed56d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
@@ -32,6 +32,7 @@ import javax.ws.rs.core.MediaType;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
+import com.sun.jersey.api.client.filter.LoggingFilter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.NodeHealthScriptRunner;
@@ -245,6 +246,7 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
Application app2 = new MockApp(2);
nmContext.getApplications().put(app2.getAppId(), app2);
addAppContainers(app2);
+ client().addFilter(new LoggingFilter());
ClientResponse response = r.path("ws").path("v1").path("node").path(path)
.accept(media).get(ClientResponse.class);
@@ -412,6 +414,7 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
Application app2 = new MockApp(2);
nmContext.getApplications().put(app2.getAppId(), app2);
addAppContainers(app2);
+ client().addFilter(new LoggingFilter(System.out));
for (String id : hash.keySet()) {
ClientResponse response = r.path("ws").path("v1").path("node")
@@ -471,12 +474,16 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
WebServicesTestUtils.getXmlInt(element, "totalMemoryNeededMB"),
WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"),
WebServicesTestUtils.getXmlString(element, "containerLogsLink"));
+ // verify that the container log files element exists
+ assertTrue("containerLogFiles missing",
+ WebServicesTestUtils.getXmlString(element, "containerLogFiles")
+ != null);
}
}
public void verifyNodeContainerInfo(JSONObject info, Container cont)
throws JSONException, Exception {
- assertEquals("incorrect number of elements", 9, info.length());
+ assertEquals("incorrect number of elements", 10, info.length());
verifyNodeContainerInfoGeneric(cont, info.getString("id"),
info.getString("state"), info.getString("user"),
@@ -484,6 +491,9 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
info.getString("nodeId"), info.getInt("totalMemoryNeededMB"),
info.getInt("totalVCoresNeeded"),
info.getString("containerLogsLink"));
+ // verify that the container log files element exists
+ assertTrue("containerLogFiles missing",
+ info.getJSONArray("containerLogFiles") != null);
}
public void verifyNodeContainerInfoGeneric(Container cont, String id,
@@ -514,5 +524,4 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
cont.getUser());
assertTrue("containerLogsLink wrong", logsLink.contains(shortLink));
}
-
}