You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2016/07/19 22:01:50 UTC
[10/50] hadoop git commit: YARN-5200. Enhanced "yarn logs" to be able
to get a list of containers whose logs are aggregated via a
"show_container_log_info" option. Contributed by Xuan Gong.
YARN-5200. Enhanced "yarn logs" to be able to get a list of containers whose logs are aggregated via a "show_container_log_info" option. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb471632
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb471632
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb471632
Branch: refs/heads/YARN-2915
Commit: eb471632349deac4b62f8dec853c8ceb64c9617a
Parents: 5614217
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Wed Jul 13 10:47:43 2016 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Wed Jul 13 10:54:15 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 321 ++++++++++++++-----
.../hadoop/yarn/client/cli/TestLogsCLI.java | 85 +++--
.../logaggregation/AggregatedLogFormat.java | 13 +-
.../yarn/logaggregation/LogCLIHelpers.java | 97 +++++-
4 files changed, 392 insertions(+), 124 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb471632/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 79401b1..31395be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -78,14 +78,30 @@ public class LogsCLI extends Configured implements Tool {
private static final String APP_OWNER_OPTION = "appOwner";
private static final String AM_CONTAINER_OPTION = "am";
private static final String CONTAINER_LOG_FILES = "logFiles";
- private static final String SHOW_META_INFO = "show_meta_info";
private static final String LIST_NODES_OPTION = "list_nodes";
+ private static final String SHOW_APPLICATION_LOG_INFO
+ = "show_application_log_info";
+ private static final String SHOW_CONTAINER_LOG_INFO
+ = "show_container_log_info";
private static final String OUT_OPTION = "out";
private static final String SIZE_OPTION = "size";
public static final String HELP_CMD = "help";
+ private PrintStream outStream = System.out;
+ private YarnClient yarnClient = null;
@Override
public int run(String[] args) throws Exception {
+ try {
+ yarnClient = createYarnClient();
+ return runCommand(args);
+ } finally {
+ if (yarnClient != null) {
+ yarnClient.close();
+ }
+ }
+ }
+
+ private int runCommand(String[] args) throws Exception {
Options opts = createCommandOpts();
Options printOpts = createPrintOpts(opts);
if (args.length < 1) {
@@ -102,8 +118,9 @@ public class LogsCLI extends Configured implements Tool {
String nodeAddress = null;
String appOwner = null;
boolean getAMContainerLogs = false;
- boolean showMetaInfo = false;
boolean nodesList = false;
+ boolean showApplicationLogInfo = false;
+ boolean showContainerLogInfo = false;
String[] logFiles = null;
List<String> amContainersList = new ArrayList<String>();
String localDir = null;
@@ -115,9 +132,11 @@ public class LogsCLI extends Configured implements Tool {
nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION);
appOwner = commandLine.getOptionValue(APP_OWNER_OPTION);
getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION);
- showMetaInfo = commandLine.hasOption(SHOW_META_INFO);
nodesList = commandLine.hasOption(LIST_NODES_OPTION);
localDir = commandLine.getOptionValue(OUT_OPTION);
+ showApplicationLogInfo = commandLine.hasOption(
+ SHOW_APPLICATION_LOG_INFO);
+ showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO);
if (getAMContainerLogs) {
try {
amContainersList = parseAMContainer(commandLine, printOpts);
@@ -172,6 +191,12 @@ public class LogsCLI extends Configured implements Tool {
}
}
+ if (showApplicationLogInfo && showContainerLogInfo) {
+ System.err.println("Invalid options. Can only accept one of "
+ + "show_application_log_info/show_container_log_info.");
+ return -1;
+ }
+
LogCLIHelpers logCliHelper = new LogCLIHelpers();
logCliHelper.setConf(getConf());
@@ -215,14 +240,17 @@ public class LogsCLI extends Configured implements Tool {
isApplicationFinished(appState), appOwner, nodeAddress, null,
containerIdStr, localDir, logs, bytes);
- if (showMetaInfo) {
- return showMetaInfo(request, logCliHelper);
+ if (showContainerLogInfo) {
+ return showContainerLogInfo(request, logCliHelper);
}
if (nodesList) {
return showNodeLists(request, logCliHelper);
}
+ if (showApplicationLogInfo) {
+ return showApplicationLogInfo(request, logCliHelper);
+ }
// To get am logs
if (getAMContainerLogs) {
return fetchAMContainerLogs(request, amContainersList,
@@ -246,21 +274,15 @@ public class LogsCLI extends Configured implements Tool {
private ApplicationReport getApplicationReport(ApplicationId appId)
throws IOException, YarnException {
- YarnClient yarnClient = createYarnClient();
-
- try {
- return yarnClient.getApplicationReport(appId);
- } finally {
- yarnClient.close();
- }
+ return yarnClient.getApplicationReport(appId);
}
@VisibleForTesting
protected YarnClient createYarnClient() {
- YarnClient yarnClient = YarnClient.createYarnClient();
- yarnClient.init(getConf());
- yarnClient.start();
- return yarnClient;
+ YarnClient client = YarnClient.createYarnClient();
+ client.init(getConf());
+ client.start();
+ return client;
}
public static void main(String[] args) throws Exception {
@@ -272,7 +294,7 @@ public class LogsCLI extends Configured implements Tool {
}
private void printHelpMessage(Options options) {
- System.out.println("Retrieve logs for completed YARN applications.");
+ outStream.println("Retrieve logs for YARN applications.");
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("yarn logs -applicationId <application ID> [OPTIONS]",
new Options());
@@ -336,9 +358,9 @@ public class LogsCLI extends Configured implements Tool {
return false;
}
- private List<String> getContainerLogFiles(Configuration conf,
+ private List<PerLogFileInfo> getContainerLogFiles(Configuration conf,
String containerIdStr, String nodeHttpAddress) throws IOException {
- List<String> logFiles = new ArrayList<>();
+ List<PerLogFileInfo> logFileInfos = new ArrayList<>();
Client webServiceClient = Client.create();
try {
WebResource webResource = webServiceClient
@@ -354,7 +376,9 @@ public class LogsCLI extends Configured implements Tool {
JSONObject json = response.getEntity(JSONObject.class);
JSONArray array = json.getJSONArray("containerLogInfo");
for (int i = 0; i < array.length(); i++) {
- logFiles.add(array.getJSONObject(i).getString("fileName"));
+ String fileName = array.getJSONObject(i).getString("fileName");
+ String fileSize = array.getJSONObject(i).getString("fileSize");
+ logFileInfos.add(new PerLogFileInfo(fileName, fileSize));
}
} catch (Exception e) {
System.err.println("Unable to parse json from webservice. Error:");
@@ -367,7 +391,7 @@ public class LogsCLI extends Configured implements Tool {
System.err.println("Unable to fetch log files list");
throw new IOException(ex);
}
- return logFiles;
+ return logFileInfos;
}
@Private
@@ -378,15 +402,26 @@ public class LogsCLI extends Configured implements Tool {
String containerIdStr = request.getContainerId().toString();
String localDir = request.getOutputLocalDir();
String nodeHttpAddress = request.getNodeHttpAddress();
+ if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
+ System.err.println("Can not get the logs for the container: "
+ + containerIdStr);
+ System.err.println("The node http address is required to get container "
+ + "logs for the Running application.");
+ return -1;
+ }
String nodeId = request.getNodeId();
PrintStream out = logCliHelper.createPrintStream(localDir, nodeId,
containerIdStr);
try {
// fetch all the log files for the container
// filter the log files based on the given --logFiles pattern
- List<String> allLogs=
+ List<PerLogFileInfo> allLogFileInfos=
getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress);
- List<String> matchedFiles = getMatchedLogFiles(request, allLogs);
+ List<String> fileNames = new ArrayList<String>();
+ for (PerLogFileInfo fileInfo : allLogFileInfos) {
+ fileNames.add(fileInfo.getFileName());
+ }
+ List<String> matchedFiles = getMatchedLogFiles(request, fileNames);
if (matchedFiles.isEmpty()) {
System.err.println("Can not find any log file matching the pattern: "
+ request.getLogTypes() + " for the container: " + containerIdStr
@@ -397,8 +432,8 @@ public class LogsCLI extends Configured implements Tool {
newOptions.setLogTypes(matchedFiles);
Client webServiceClient = Client.create();
- String containerString = "\n\nContainer: " + containerIdStr + " on "
- + nodeId;
+ String containerString = String.format(
+ LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerIdStr, nodeId);
out.println(containerString);
out.println(StringUtils.repeat("=", containerString.length()));
boolean foundAnyLogs = false;
@@ -477,13 +512,8 @@ public class LogsCLI extends Configured implements Tool {
@VisibleForTesting
public ContainerReport getContainerReport(String containerIdStr)
throws YarnException, IOException {
- YarnClient yarnClient = createYarnClient();
- try {
- return yarnClient.getContainerReport(
- ContainerId.fromString(containerIdStr));
- } finally {
- yarnClient.close();
- }
+ return yarnClient.getContainerReport(
+ ContainerId.fromString(containerIdStr));
}
private boolean isApplicationFinished(YarnApplicationState appState) {
@@ -508,8 +538,10 @@ public class LogsCLI extends Configured implements Tool {
for (JSONObject amContainer : amContainersList) {
ContainerLogsRequest amRequest = new ContainerLogsRequest(request);
amRequest.setContainerId(amContainer.getString("containerId"));
- amRequest.setNodeHttpAddress(
- amContainer.getString("nodeHttpAddress"));
+ String httpAddress = amContainer.getString("nodeHttpAddress");
+ if (httpAddress != null && !httpAddress.isEmpty()) {
+ amRequest.setNodeHttpAddress(httpAddress);
+ }
amRequest.setNodeId(amContainer.getString("nodeId"));
requests.add(amRequest);
}
@@ -545,8 +577,8 @@ public class LogsCLI extends Configured implements Tool {
for (ContainerLogsRequest amRequest : requests) {
outputAMContainerLogs(amRequest, conf, logCliHelper);
}
- System.out.println();
- System.out.println("Specified ALL for -am option. "
+ outStream.println();
+ outStream.println("Specified ALL for -am option. "
+ "Printed logs for all am containers.");
} else {
for (String amContainer : amContainers) {
@@ -602,15 +634,13 @@ public class LogsCLI extends Configured implements Tool {
}
}
- private int showMetaInfo(ContainerLogsRequest request,
- LogCLIHelpers logCliHelper) throws IOException {
+ private int showContainerLogInfo(ContainerLogsRequest request,
+ LogCLIHelpers logCliHelper) throws IOException, YarnException {
if (!request.isAppFinished()) {
- System.err.println("The -show_meta_info command can be only used "
- + "with finished applications");
- return -1;
+ return printContainerInfoFromRunningApplication(request);
} else {
- logCliHelper.printLogMetadata(request, System.out, System.err);
- return 0;
+ return logCliHelper.printAContainerLogMetadata(
+ request, System.out, System.err);
}
}
@@ -626,6 +656,33 @@ public class LogsCLI extends Configured implements Tool {
}
}
+ private int showApplicationLogInfo(ContainerLogsRequest request,
+ LogCLIHelpers logCliHelper) throws IOException, YarnException {
+ String appState = "Application State: "
+ + (request.isAppFinished() ? "Completed." : "Running.");
+ if (!request.isAppFinished()) {
+ List<ContainerReport> reports =
+ getContainerReportsFromRunningApplication(request);
+ List<ContainerReport> filterReports = filterContainersInfo(
+ request, reports);
+ if (filterReports.isEmpty()) {
+ System.err.println("Can not find any containers for the application:"
+ + request.getAppId() + ".");
+ return -1;
+ }
+ outStream.println(appState);
+ for (ContainerReport report : filterReports) {
+ outStream.println(String.format(LogCLIHelpers.CONTAINER_ON_NODE_PATTERN,
+ report.getContainerId(), report.getAssignedNode()));
+ }
+ return 0;
+ } else {
+ outStream.println(appState);
+ logCliHelper.printContainersList(request, System.out, System.err);
+ return 0;
+ }
+ }
+
private Options createCommandOpts() {
Options opts = new Options();
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
@@ -661,13 +718,16 @@ public class LogsCLI extends Configured implements Tool {
logFileOpt.setArgs(Option.UNLIMITED_VALUES);
logFileOpt.setArgName("Log File Name");
opts.addOption(logFileOpt);
- opts.addOption(SHOW_META_INFO, false, "Show the log metadata, "
+ opts.addOption(SHOW_CONTAINER_LOG_INFO, false,
+ "Show the container log metadata, "
+ "including log-file names, the size of the log files. "
+ "You can combine this with --containerId to get log metadata for "
+ "the specific container, or with --nodeAddress to get log metadata "
- + "for all the containers on the specific NodeManager. "
- + "Currently, this option can only be used for finished "
- + "applications.");
+ + "for all the containers on the specific NodeManager.");
+ opts.addOption(SHOW_APPLICATION_LOG_INFO, false, "Show the "
+ + "containerIds which belong to the specific Application. "
+ + "You can combine this with --nodeAddress to get containerIds "
+ + "for all the containers on the specific NodeManager.");
opts.addOption(LIST_NODES_OPTION, false,
"Show the list of nodes that successfully aggregated logs. "
+ "This option can only be used with finished applications.");
@@ -695,8 +755,9 @@ public class LogsCLI extends Configured implements Tool {
printOpts.addOption(commandOpts.getOption(APP_OWNER_OPTION));
printOpts.addOption(commandOpts.getOption(AM_CONTAINER_OPTION));
printOpts.addOption(commandOpts.getOption(CONTAINER_LOG_FILES));
- printOpts.addOption(commandOpts.getOption(SHOW_META_INFO));
printOpts.addOption(commandOpts.getOption(LIST_NODES_OPTION));
+ printOpts.addOption(commandOpts.getOption(SHOW_APPLICATION_LOG_INFO));
+ printOpts.addOption(commandOpts.getOption(SHOW_CONTAINER_LOG_INFO));
printOpts.addOption(commandOpts.getOption(OUT_OPTION));
printOpts.addOption(commandOpts.getOption(SIZE_OPTION));
return printOpts;
@@ -801,12 +862,14 @@ public class LogsCLI extends Configured implements Tool {
// the ContainerReport. In the containerReport, we could get
// nodeAddress and nodeHttpAddress
ContainerReport report = getContainerReport(containerIdStr);
- nodeHttpAddress =
- report.getNodeHttpAddress().replaceFirst(
- WebAppUtils.getHttpSchemePrefix(getConf()), "");
+ nodeHttpAddress = report.getNodeHttpAddress();
+ if (nodeHttpAddress != null && !nodeHttpAddress.isEmpty()) {
+ nodeHttpAddress = nodeHttpAddress.replaceFirst(
+ WebAppUtils.getHttpSchemePrefix(getConf()), "");
+ request.setNodeHttpAddress(nodeHttpAddress);
+ }
nodeId = report.getAssignedNode().toString();
request.setNodeId(nodeId);
- request.setNodeHttpAddress(nodeHttpAddress);
} catch (IOException | YarnException ex) {
if (isAppFinished) {
return printContainerLogsForFinishedApplicationWithoutNodeId(
@@ -942,32 +1005,140 @@ public class LogsCLI extends Configured implements Tool {
ContainerLogsRequest options) throws YarnException, IOException {
List<ContainerLogsRequest> newOptionsList =
new ArrayList<ContainerLogsRequest>();
- YarnClient yarnClient = createYarnClient();
- try {
- List<ApplicationAttemptReport> attempts =
- yarnClient.getApplicationAttempts(options.getAppId());
- for (ApplicationAttemptReport attempt : attempts) {
- List<ContainerReport> containers = yarnClient.getContainers(
- attempt.getApplicationAttemptId());
- for (ContainerReport container : containers) {
- ContainerLogsRequest newOptions = new ContainerLogsRequest(options);
- newOptions.setContainerId(container.getContainerId().toString());
- newOptions.setNodeId(container.getAssignedNode().toString());
- newOptions.setNodeHttpAddress(container.getNodeHttpAddress()
- .replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""));
- // if we do not specify the value for CONTAINER_LOG_FILES option,
- // we will only output syslog
- List<String> logFiles = newOptions.getLogTypes();
- if (logFiles == null || logFiles.isEmpty()) {
- logFiles = Arrays.asList("syslog");
- newOptions.setLogTypes(logFiles);
+ List<ContainerReport> reports =
+ getContainerReportsFromRunningApplication(options);
+ for (ContainerReport container : reports) {
+ ContainerLogsRequest newOptions = new ContainerLogsRequest(options);
+ newOptions.setContainerId(container.getContainerId().toString());
+ newOptions.setNodeId(container.getAssignedNode().toString());
+ String httpAddress = container.getNodeHttpAddress();
+ if (httpAddress != null && !httpAddress.isEmpty()) {
+ newOptions.setNodeHttpAddress(httpAddress
+ .replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""));
+ }
+ // if we do not specify the value for CONTAINER_LOG_FILES option,
+ // we will only output syslog
+ List<String> logFiles = newOptions.getLogTypes();
+ if (logFiles == null || logFiles.isEmpty()) {
+ logFiles = Arrays.asList("syslog");
+ newOptions.setLogTypes(logFiles);
+ }
+ newOptionsList.add(newOptions);
+ }
+ return newOptionsList;
+ }
+
+ private List<ContainerReport> getContainerReportsFromRunningApplication(
+ ContainerLogsRequest options) throws YarnException, IOException {
+ List<ContainerReport> reports = new ArrayList<ContainerReport>();
+ List<ApplicationAttemptReport> attempts =
+ yarnClient.getApplicationAttempts(options.getAppId());
+ for (ApplicationAttemptReport attempt : attempts) {
+ List<ContainerReport> containers = yarnClient.getContainers(
+ attempt.getApplicationAttemptId());
+ reports.addAll(containers);
+ }
+ return reports;
+ }
+
+ // filter the containerReports based on the nodeId and ContainerId
+ private List<ContainerReport> filterContainersInfo(
+ ContainerLogsRequest options, List<ContainerReport> containers) {
+ List<ContainerReport> filterReports = new ArrayList<ContainerReport>(
+ containers);
+ String nodeId = options.getNodeId();
+ boolean filterBasedOnNodeId = (nodeId != null && !nodeId.isEmpty());
+ String containerId = options.getContainerId();
+ boolean filterBasedOnContainerId = (containerId != null
+ && !containerId.isEmpty());
+
+ if (filterBasedOnNodeId || filterBasedOnContainerId) {
+ // filter the reports based on the containerId and.or nodeId
+ for(ContainerReport report : containers) {
+ if (filterBasedOnContainerId) {
+ if (!report.getContainerId().toString()
+ .equalsIgnoreCase(containerId)) {
+ filterReports.remove(report);
+ }
+ }
+
+ if (filterBasedOnNodeId) {
+ if (!report.getAssignedNode().toString().equalsIgnoreCase(nodeId)) {
+ filterReports.remove(report);
}
- newOptionsList.add(newOptions);
}
}
- return newOptionsList;
- } finally {
- yarnClient.close();
+ }
+ return filterReports;
+ }
+
+ private int printContainerInfoFromRunningApplication(
+ ContainerLogsRequest options) throws YarnException, IOException {
+ String containerIdStr = options.getContainerId();
+ String nodeIdStr = options.getNodeId();
+ List<ContainerReport> reports =
+ getContainerReportsFromRunningApplication(options);
+ List<ContainerReport> filteredReports = filterContainersInfo(
+ options, reports);
+ if (filteredReports.isEmpty()) {
+ StringBuilder sb = new StringBuilder();
+ if (containerIdStr != null && !containerIdStr.isEmpty()) {
+ sb.append("Trying to get container with ContainerId: "
+ + containerIdStr + "\n");
+ }
+ if (nodeIdStr != null && !nodeIdStr.isEmpty()) {
+ sb.append("Trying to get container from NodeManager: "
+ + nodeIdStr + "\n");
+ }
+ sb.append("Can not find any matched containers for the application: "
+ + options.getAppId());
+ System.err.println(sb.toString());
+ return -1;
+ }
+ for (ContainerReport report : filteredReports) {
+ String nodeId = report.getAssignedNode().toString();
+ String nodeHttpAddress = report.getNodeHttpAddress().replaceFirst(
+ WebAppUtils.getHttpSchemePrefix(getConf()), "");
+ String containerId = report.getContainerId().toString();
+ String containerString = String.format(
+ LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerId, nodeId);
+ outStream.println(containerString);
+ outStream.println(StringUtils.repeat("=", containerString.length()));
+ outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN,
+ "LogType", "LogLength");
+ outStream.println(StringUtils.repeat("=", containerString.length()));
+ List<PerLogFileInfo> infos = getContainerLogFiles(
+ getConf(), containerId, nodeHttpAddress);
+ for (PerLogFileInfo info : infos) {
+ outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN,
+ info.getFileName(), info.getFileLength());
+ }
+ }
+ return 0;
+ }
+
+ private static class PerLogFileInfo {
+ private String fileName;
+ private String fileLength;
+ public PerLogFileInfo(String fileName, String fileLength) {
+ setFileName(fileName);
+ setFileLength(fileLength);
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public void setFileName(String fileName) {
+ this.fileName = fileName;
+ }
+
+ public String getFileLength() {
+ return fileLength;
+ }
+
+ public void setFileLength(String fileLength) {
+ this.fileLength = fileLength;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb471632/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index 34369ef..cb9a663 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -182,7 +182,7 @@ public class TestLogsCLI {
assertTrue(exitCode == -1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
- pw.println("Retrieve logs for completed YARN applications.");
+ pw.println("Retrieve logs for YARN applications.");
pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]");
pw.println();
pw.println("general options are:");
@@ -217,14 +217,18 @@ public class TestLogsCLI {
pw.println(" container logs. The container logs will");
pw.println(" be stored based on the node the container");
pw.println(" ran on.");
- pw.println(" -show_meta_info Show the log metadata, including log-file");
- pw.println(" names, the size of the log files. You can");
- pw.println(" combine this with --containerId to get");
- pw.println(" log metadata for the specific container,");
- pw.println(" or with --nodeAddress to get log metadata");
- pw.println(" for all the containers on the specific");
- pw.println(" NodeManager. Currently, this option can");
- pw.println(" only be used for finished applications.");
+ pw.println(" -show_application_log_info Show the containerIds which belong to the");
+ pw.println(" specific Application. You can combine");
+ pw.println(" this with --nodeAddress to get");
+ pw.println(" containerIds for all the containers on");
+ pw.println(" the specific NodeManager.");
+ pw.println(" -show_container_log_info Show the container log metadata,");
+ pw.println(" including log-file names, the size of the");
+ pw.println(" log files. You can combine this with");
+ pw.println(" --containerId to get log metadata for the");
+ pw.println(" specific container, or with --nodeAddress");
+ pw.println(" to get log metadata for all the");
+ pw.println(" containers on the specific NodeManager.");
pw.println(" -size <size> Prints the log file's first 'n' bytes or");
pw.println(" the last 'n' bytes. Use negative values");
pw.println(" as bytes to read from the end and");
@@ -698,9 +702,8 @@ public class TestLogsCLI {
"-applicationId", appTest.toString()});
assertTrue(exitCode == -1);
assertTrue(sysErrStream.toString().contains(
- "Guessed logs' owner is " + priorityUser + " and current user "
- + UserGroupInformation.getCurrentUser().getUserName()
- + " does not have permission to access"));
+ "Can not find the logs for the application: "
+ + appTest.toString()));
sysErrStream.reset();
} finally {
fs.delete(new Path(remoteLogRootDir), true);
@@ -842,49 +845,77 @@ public class TestLogsCLI {
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
cli.setConf(configuration);
- cli.run(new String[] { "-applicationId", appId.toString(),
- "-show_meta_info" });
+ int result = cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_container_log_info", "-show_application_log_info"});
+ assertTrue(result == -1);
+ assertTrue(sysErrStream.toString().contains("Invalid options. "
+ + "Can only accept one of show_application_log_info/"
+ + "show_container_log_info."));
+
+ cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_container_log_info"});
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000001 on localhost_"));
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000002 on localhost_"));
assertTrue(sysOutStream.toString().contains(
- "LogType:syslog"));
+ "syslog"));
assertTrue(sysOutStream.toString().contains(
- "LogLength:43"));
+ "43"));
sysOutStream.reset();
- cli.run(new String[] { "-applicationId", appId.toString(),
- "-show_meta_info", "-containerId", "container_0_0001_01_000001" });
+ cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_container_log_info", "-containerId",
+ "container_0_0001_01_000001"});
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000001 on localhost_"));
assertFalse(sysOutStream.toString().contains(
"Container: container_0_0001_01_000002 on localhost_"));
assertTrue(sysOutStream.toString().contains(
- "LogType:syslog"));
+ "syslog"));
assertTrue(sysOutStream.toString().contains(
- "LogLength:43"));
+ "43"));
sysOutStream.reset();
- cli.run(new String[] { "-applicationId", appId.toString(),
- "-show_meta_info", "-nodeAddress", "localhost" });
+ cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_container_log_info", "-nodeAddress", "localhost"});
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000001 on localhost_"));
assertTrue(sysOutStream.toString().contains(
"Container: container_0_0001_01_000002 on localhost_"));
assertTrue(sysOutStream.toString().contains(
- "LogType:syslog"));
+ "syslog"));
assertTrue(sysOutStream.toString().contains(
- "LogLength:43"));
+ "43"));
sysOutStream.reset();
- cli.run(new String[] { "-applicationId", appId.toString(),
- "-show_meta_info", "-nodeAddress", "localhost", "-containerId",
- "container_1234" });
+ cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_container_log_info", "-nodeAddress", "localhost",
+ "-containerId", "container_1234"});
assertTrue(sysErrStream.toString().contains(
"Invalid ContainerId specified"));
sysErrStream.reset();
+ cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_application_log_info"});
+ assertTrue(sysOutStream.toString().contains(
+ "Application State: Completed."));
+ assertTrue(sysOutStream.toString().contains(
+ "container_0_0001_01_000001 on localhost"));
+ assertTrue(sysOutStream.toString().contains(
+ "container_0_0001_01_000002 on localhost"));
+ sysOutStream.reset();
+
+ cli.run(new String[] {"-applicationId", appId.toString(),
+ "-show_application_log_info", "-nodeAddress", "localhost"});
+ assertTrue(sysOutStream.toString().contains(
+ "Application State: Completed."));
+ assertTrue(sysOutStream.toString().contains(
+ "container_0_0001_01_000001 on localhost"));
+ assertTrue(sysOutStream.toString().contains(
+ "container_0_0001_01_000002 on localhost"));
+ sysOutStream.reset();
+
fs.delete(new Path(remoteLogRootDir), true);
fs.delete(new Path(rootLogDir), true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb471632/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 7508dd5..3c4f835 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -49,6 +49,7 @@ import org.apache.commons.io.input.BoundedInputStream;
import org.apache.commons.io.output.WriterOutputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
@@ -959,25 +960,21 @@ public class AggregatedLogFormat {
}
@Private
- public static String readContainerMetaDataAndSkipData(
+ public static Pair<String, String> readContainerMetaDataAndSkipData(
DataInputStream valueStream, PrintStream out) throws IOException {
String fileType = valueStream.readUTF();
String fileLengthStr = valueStream.readUTF();
long fileLength = Long.parseLong(fileLengthStr);
- if (out != null) {
- out.print("LogType:");
- out.println(fileType);
- out.print("LogLength:");
- out.println(fileLengthStr);
- }
+ Pair<String, String> logMeta = new Pair<String, String>(
+ fileType, fileLengthStr);
long totalSkipped = 0;
long currSkipped = 0;
while (currSkipped != -1 && totalSkipped < fileLength) {
currSkipped = valueStream.skip(fileLength - totalSkipped);
totalSkipped += currSkipped;
}
- return fileType;
+ return logMeta;
}
public void close() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb471632/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index 4de11a5..ef56504 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -32,6 +32,7 @@ import java.util.List;
import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -50,6 +51,11 @@ import com.google.common.annotations.VisibleForTesting;
public class LogCLIHelpers implements Configurable {
+ public static final String PER_LOG_FILE_INFO_PATTERN =
+ "%20s\t%20s" + System.getProperty("line.separator");
+ public static final String CONTAINER_ON_NODE_PATTERN =
+ "Container: %s on %s";
+
private Configuration conf;
@Private
@@ -151,8 +157,8 @@ public class LogCLIHelpers implements Configurable {
AggregatedLogFormat.LogReader reader = null;
PrintStream out = createPrintStream(localDir, fileName, containerId);
try {
- String containerString = "\n\nContainer: " + containerId + " on "
- + thisNodeFile.getPath().getName();
+ String containerString = String.format(CONTAINER_ON_NODE_PATTERN,
+ containerId, thisNodeFile.getPath().getName());
out.println(containerString);
out.println(StringUtils.repeat("=", containerString.length()));
reader =
@@ -219,8 +225,10 @@ public class LogCLIHelpers implements Configurable {
thisNodeFile.getPath());
out = createPrintStream(localDir, thisNodeFile.getPath().getName(),
containerId);
- out.println(containerId + " on " + thisNodeFile.getPath().getName());
- out.println(StringUtils.repeat("=", containerId.length()));
+ String containerString = String.format(CONTAINER_ON_NODE_PATTERN,
+ containerId, thisNodeFile.getPath().getName());
+ out.println(containerString);
+ out.println(StringUtils.repeat("=", containerString.length()));
if (logType == null || logType.isEmpty()) {
if (dumpAContainerLogs(containerId, reader, out,
thisNodeFile.getModificationTime(), options.getBytes()) > -1) {
@@ -355,9 +363,9 @@ public class LogCLIHelpers implements Configurable {
PrintStream out = createPrintStream(localDir,
thisNodeFile.getPath().getName(), key.toString());
try {
- String containerString =
- "\n\nContainer: " + key + " on "
- + thisNodeFile.getPath().getName();
+ String containerString = String.format(
+ CONTAINER_ON_NODE_PATTERN, key,
+ thisNodeFile.getPath().getName());
out.println(containerString);
out.println(StringUtils.repeat("=", containerString.length()));
while (true) {
@@ -400,7 +408,7 @@ public class LogCLIHelpers implements Configurable {
}
@Private
- public void printLogMetadata(ContainerLogsRequest options,
+ public int printAContainerLogMetadata(ContainerLogsRequest options,
PrintStream out, PrintStream err)
throws IOException {
ApplicationId appId = options.getAppId();
@@ -413,7 +421,7 @@ public class LogCLIHelpers implements Configurable {
RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(
appId, appOwner);
if (nodeFiles == null) {
- return;
+ return -1;
}
boolean foundAnyLogs = false;
while (nodeFiles.hasNext()) {
@@ -434,16 +442,21 @@ public class LogCLIHelpers implements Configurable {
valueStream = reader.next(key);
while (valueStream != null) {
if (getAllContainers || (key.toString().equals(containerIdStr))) {
- String containerString =
- "\n\nContainer: " + key + " on "
- + thisNodeFile.getPath().getName();
+ String containerString = String.format(CONTAINER_ON_NODE_PATTERN,
+ key, thisNodeFile.getPath().getName());
out.println(containerString);
out.println("Log Upload Time:"
+ thisNodeFile.getModificationTime());
out.println(StringUtils.repeat("=", containerString.length()));
+ out.printf(PER_LOG_FILE_INFO_PATTERN, "LogType", "LogLength");
+ out.println(StringUtils.repeat("=", containerString.length()));
while (true) {
try {
- LogReader.readContainerMetaDataAndSkipData(valueStream, out);
+ Pair<String, String> logMeta =
+ LogReader.readContainerMetaDataAndSkipData(
+ valueStream, out);
+ out.printf(PER_LOG_FILE_INFO_PATTERN,
+ logMeta.getFirst(), logMeta.getSecond());
} catch (EOFException eof) {
break;
}
@@ -473,7 +486,9 @@ public class LogCLIHelpers implements Configurable {
err.println("Can not find log metadata for container: "
+ containerIdStr);
}
+ return -1;
}
+ return 0;
}
@Private
@@ -501,6 +516,60 @@ public class LogCLIHelpers implements Configurable {
}
}
+ @Private
+ public void printContainersList(ContainerLogsRequest options,
+ PrintStream out, PrintStream err) throws IOException {
+ ApplicationId appId = options.getAppId();
+ String appOwner = options.getAppOwner();
+ String nodeId = options.getNodeId();
+ String nodeIdStr = (nodeId == null) ? null
+ : LogAggregationUtils.getNodeString(nodeId);
+ RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(
+ appId, appOwner);
+ if (nodeFiles == null) {
+ return;
+ }
+ boolean foundAnyLogs = false;
+ while (nodeFiles.hasNext()) {
+ FileStatus thisNodeFile = nodeFiles.next();
+ if (nodeIdStr != null) {
+ if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) {
+ continue;
+ }
+ }
+ if (!thisNodeFile.getPath().getName()
+ .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
+ AggregatedLogFormat.LogReader reader =
+ new AggregatedLogFormat.LogReader(getConf(),
+ thisNodeFile.getPath());
+ try {
+ DataInputStream valueStream;
+ LogKey key = new LogKey();
+ valueStream = reader.next(key);
+ while (valueStream != null) {
+ out.println(String.format(CONTAINER_ON_NODE_PATTERN, key,
+ thisNodeFile.getPath().getName()));
+ foundAnyLogs = true;
+ // Next container
+ key = new LogKey();
+ valueStream = reader.next(key);
+ }
+ } finally {
+ reader.close();
+ }
+ }
+ }
+ if (!foundAnyLogs) {
+ if (nodeId != null) {
+ err.println("Can not find information for any containers on "
+ + nodeId);
+ } else {
+ err.println("Can not find any container information for "
+ + "the application: " + appId);
+ }
+ }
+ }
+
private RemoteIterator<FileStatus> getRemoteNodeFileDir(ApplicationId appId,
String appOwner) throws IOException {
Path remoteAppLogDir = getRemoteAppLogDir(appId, appOwner);
@@ -621,7 +690,7 @@ public class LogCLIHelpers implements Configurable {
while (true) {
try {
String logFile = LogReader.readContainerMetaDataAndSkipData(
- valueStream, null);
+ valueStream, null).getFirst();
logTypes.add(logFile);
} catch (EOFException eof) {
break;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org