You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2016/08/09 22:33:56 UTC
[14/50] [abbrv] hadoop git commit: YARN-5470. Addedum to
differentiate exactly matching of log-files with regex in yarn log CLI.
Contributed by Xuan Gong.
YARN-5470. Addedum to differentiate exactly matching of log-files with regex in yarn log CLI. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83a2ffec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83a2ffec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83a2ffec
Branch: refs/heads/YARN-3368
Commit: 83a2ffec408fc3ebb89ab13ab4788d9f02fae33b
Parents: 58e1523
Author: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Authored: Mon Aug 8 14:28:07 2016 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Mon Aug 8 14:28:07 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 62 ++++++---
.../hadoop/yarn/client/cli/TestLogsCLI.java | 129 +++++++++++--------
2 files changed, 122 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a2ffec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index ec045b2..908d379 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -84,6 +84,8 @@ public class LogsCLI extends Configured implements Tool {
private static final String APP_OWNER_OPTION = "appOwner";
private static final String AM_CONTAINER_OPTION = "am";
private static final String PER_CONTAINER_LOG_FILES_OPTION = "log_files";
+ private static final String PER_CONTAINER_LOG_FILES_REGEX_OPTION
+ = "log_files_pattern";
private static final String LIST_NODES_OPTION = "list_nodes";
private static final String SHOW_APPLICATION_LOG_INFO
= "show_application_log_info";
@@ -91,7 +93,6 @@ public class LogsCLI extends Configured implements Tool {
= "show_container_log_info";
private static final String OUT_OPTION = "out";
private static final String SIZE_OPTION = "size";
- private static final String REGEX_OPTION = "regex";
public static final String HELP_CMD = "help";
private PrintStream outStream = System.out;
private YarnClient yarnClient = null;
@@ -130,6 +131,7 @@ public class LogsCLI extends Configured implements Tool {
boolean showContainerLogInfo = false;
boolean useRegex = false;
String[] logFiles = null;
+ String[] logFilesRegex = null;
List<String> amContainersList = new ArrayList<String>();
String localDir = null;
long bytes = Long.MAX_VALUE;
@@ -145,7 +147,6 @@ public class LogsCLI extends Configured implements Tool {
showApplicationLogInfo = commandLine.hasOption(
SHOW_APPLICATION_LOG_INFO);
showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO);
- useRegex = commandLine.hasOption(REGEX_OPTION);
if (getAMContainerLogs) {
try {
amContainersList = parseAMContainer(commandLine, printOpts);
@@ -157,6 +158,11 @@ public class LogsCLI extends Configured implements Tool {
if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_OPTION)) {
logFiles = commandLine.getOptionValues(PER_CONTAINER_LOG_FILES_OPTION);
}
+ if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_REGEX_OPTION)) {
+ logFilesRegex = commandLine.getOptionValues(
+ PER_CONTAINER_LOG_FILES_REGEX_OPTION);
+ useRegex = true;
+ }
if (commandLine.hasOption(SIZE_OPTION)) {
bytes = Long.parseLong(commandLine.getOptionValue(SIZE_OPTION));
}
@@ -206,6 +212,12 @@ public class LogsCLI extends Configured implements Tool {
return -1;
}
+ if (logFiles != null && logFiles.length > 0 && logFilesRegex != null
+ && logFilesRegex.length > 0) {
+ System.err.println("Invalid options. Can only accept one of "
+ + "log_files/log_files_pattern.");
+ return -1;
+ }
if (localDir != null) {
File file = new File(localDir);
if (file.exists() && file.isFile()) {
@@ -248,10 +260,12 @@ public class LogsCLI extends Configured implements Tool {
}
Set<String> logs = new HashSet<String>();
- if (fetchAllLogFiles(logFiles, useRegex)) {
+ if (fetchAllLogFiles(logFiles, logFilesRegex)) {
logs.add("ALL");
} else if (logFiles != null && logFiles.length > 0) {
logs.addAll(Arrays.asList(logFiles));
+ } else if (logFilesRegex != null && logFilesRegex.length > 0) {
+ logs.addAll(Arrays.asList(logFilesRegex));
}
ContainerLogsRequest request = new ContainerLogsRequest(appId,
@@ -366,18 +380,28 @@ public class LogsCLI extends Configured implements Tool {
return amContainersList;
}
- private boolean fetchAllLogFiles(String[] logFiles, boolean useRegex) {
+ private boolean fetchAllLogFiles(String[] logFiles, String[] logFilesRegex) {
- // If no value is specified for the PER_CONTAINER_LOG_FILES_OPTION option,
+ // If no value is specified for the PER_CONTAINER_LOG_FILES_OPTION option
+ // and PER_CONTAINER_LOG_FILES_REGEX_OPTION
// we will assume all logs.
- if (logFiles == null || logFiles.length == 0) {
+ if ((logFiles == null || logFiles.length == 0) && (
+ logFilesRegex == null || logFilesRegex.length == 0)) {
return true;
}
- List<String> logs = Arrays.asList(logFiles);
- if (logs.contains("ALL") || logs.contains("*")||
- (logs.contains(".*") && useRegex)) {
- return true;
+ if (logFiles != null && logFiles.length > 0) {
+ List<String> logs = Arrays.asList(logFiles);
+ if (logs.contains("ALL") || logs.contains("*")) {
+ return true;
+ }
+ }
+
+ if (logFilesRegex != null && logFilesRegex.length > 0) {
+ List<String> logsRegex = Arrays.asList(logFilesRegex);
+ if (logsRegex.contains(".*")) {
+ return true;
+ }
}
return false;
@@ -753,15 +777,20 @@ public class LogsCLI extends Configured implements Tool {
opts.addOption(amOption);
Option logFileOpt = new Option(PER_CONTAINER_LOG_FILES_OPTION, true,
"Specify comma-separated value "
- + "to get exact matched log files. Use \"ALL\" or \"*\"to "
- + "fetch all the log files for the container. Specific -regex "
- + "for using java regex to find matched log files.");
+ + "to get exact matched log files. Use \"ALL\" or \"*\" to "
+ + "fetch all the log files for the container.");
logFileOpt.setValueSeparator(',');
logFileOpt.setArgs(Option.UNLIMITED_VALUES);
logFileOpt.setArgName("Log File Name");
opts.addOption(logFileOpt);
- opts.addOption(REGEX_OPTION, false, "Work with -log_files to find "
- + "matched files by using java regex.");
+ Option logFileRegexOpt = new Option(PER_CONTAINER_LOG_FILES_REGEX_OPTION,
+ true, "Specify comma-separated value "
+ + "to get matched log files by using java regex. Use \".*\" to "
+ + "fetch all the log files for the container.");
+ logFileRegexOpt.setValueSeparator(',');
+ logFileRegexOpt.setArgs(Option.UNLIMITED_VALUES);
+ logFileRegexOpt.setArgName("Log File Pattern");
+ opts.addOption(logFileRegexOpt);
opts.addOption(SHOW_CONTAINER_LOG_INFO, false,
"Show the container log metadata, "
+ "including log-file names, the size of the log files. "
@@ -804,7 +833,8 @@ public class LogsCLI extends Configured implements Tool {
printOpts.addOption(commandOpts.getOption(SHOW_CONTAINER_LOG_INFO));
printOpts.addOption(commandOpts.getOption(OUT_OPTION));
printOpts.addOption(commandOpts.getOption(SIZE_OPTION));
- printOpts.addOption(commandOpts.getOption(REGEX_OPTION));
+ printOpts.addOption(commandOpts.getOption(
+ PER_CONTAINER_LOG_FILES_REGEX_OPTION));
return printOpts;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a2ffec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index 467b204..19ddb2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -189,57 +189,70 @@ public class TestLogsCLI {
pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]");
pw.println();
pw.println("general options are:");
- pw.println(" -am <AM Containers> Prints the AM Container logs for this");
- pw.println(" application. Specify comma-separated");
- pw.println(" value to get logs for related AM");
- pw.println(" Container. For example, If we specify -am");
- pw.println(" 1,2, we will get the logs for the first");
- pw.println(" AM Container as well as the second AM");
- pw.println(" Container. To get logs for all AM");
- pw.println(" Containers, use -am ALL. To get logs for");
- pw.println(" the latest AM Container, use -am -1. By");
- pw.println(" default, it will print all available");
- pw.println(" logs. Work with -log_files to get only");
- pw.println(" specific logs.");
- pw.println(" -appOwner <Application Owner> AppOwner (assumed to be current user if");
- pw.println(" not specified)");
- pw.println(" -containerId <Container ID> ContainerId. By default, it will print");
- pw.println(" all available logs. Work with -log_files");
- pw.println(" to get only specific logs. If specified,");
- pw.println(" the applicationId can be omitted");
- pw.println(" -help Displays help for all commands.");
- pw.println(" -list_nodes Show the list of nodes that successfully");
- pw.println(" aggregated logs. This option can only be");
- pw.println(" used with finished applications.");
- pw.println(" -log_files <Log File Name> Specify comma-separated value to get");
- pw.println(" exact matched log files. Use \"ALL\" or");
- pw.println(" \"*\"to fetch all the log files for the");
- pw.println(" container. Specific -regex for using java");
- pw.println(" regex to find matched log files.");
- pw.println(" -nodeAddress <Node Address> NodeAddress in the format nodename:port");
- pw.println(" -out <Local Directory> Local directory for storing individual");
- pw.println(" container logs. The container logs will");
- pw.println(" be stored based on the node the container");
- pw.println(" ran on.");
- pw.println(" -regex Work with -log_files to find matched");
- pw.println(" files by using java regex.");
- pw.println(" -show_application_log_info Show the containerIds which belong to the");
- pw.println(" specific Application. You can combine");
- pw.println(" this with --nodeAddress to get");
- pw.println(" containerIds for all the containers on");
- pw.println(" the specific NodeManager.");
- pw.println(" -show_container_log_info Show the container log metadata,");
- pw.println(" including log-file names, the size of the");
- pw.println(" log files. You can combine this with");
- pw.println(" --containerId to get log metadata for the");
- pw.println(" specific container, or with --nodeAddress");
- pw.println(" to get log metadata for all the");
- pw.println(" containers on the specific NodeManager.");
- pw.println(" -size <size> Prints the log file's first 'n' bytes or");
- pw.println(" the last 'n' bytes. Use negative values");
- pw.println(" as bytes to read from the end and");
- pw.println(" positive values as bytes to read from the");
- pw.println(" beginning.");
+ pw.println(" -am <AM Containers> Prints the AM Container logs for");
+ pw.println(" this application. Specify");
+ pw.println(" comma-separated value to get logs");
+ pw.println(" for related AM Container. For");
+ pw.println(" example, If we specify -am 1,2,");
+ pw.println(" we will get the logs for the");
+ pw.println(" first AM Container as well as the");
+ pw.println(" second AM Container. To get logs");
+ pw.println(" for all AM Containers, use -am");
+ pw.println(" ALL. To get logs for the latest");
+ pw.println(" AM Container, use -am -1. By");
+ pw.println(" default, it will print all");
+ pw.println(" available logs. Work with");
+ pw.println(" -log_files to get only specific");
+ pw.println(" logs.");
+ pw.println(" -appOwner <Application Owner> AppOwner (assumed to be current");
+ pw.println(" user if not specified)");
+ pw.println(" -containerId <Container ID> ContainerId. By default, it will");
+ pw.println(" print all available logs. Work");
+ pw.println(" with -log_files to get only");
+ pw.println(" specific logs. If specified, the");
+ pw.println(" applicationId can be omitted");
+ pw.println(" -help Displays help for all commands.");
+ pw.println(" -list_nodes Show the list of nodes that");
+ pw.println(" successfully aggregated logs.");
+ pw.println(" This option can only be used with");
+ pw.println(" finished applications.");
+ pw.println(" -log_files <Log File Name> Specify comma-separated value to");
+ pw.println(" get exact matched log files. Use");
+ pw.println(" \"ALL\" or \"*\" to fetch all the log");
+ pw.println(" files for the container.");
+ pw.println(" -log_files_pattern <Log File Pattern> Specify comma-separated value to");
+ pw.println(" get matched log files by using");
+ pw.println(" java regex. Use \".*\" to fetch all");
+ pw.println(" the log files for the container.");
+ pw.println(" -nodeAddress <Node Address> NodeAddress in the format");
+ pw.println(" nodename:port");
+ pw.println(" -out <Local Directory> Local directory for storing");
+ pw.println(" individual container logs. The");
+ pw.println(" container logs will be stored");
+ pw.println(" based on the node the container");
+ pw.println(" ran on.");
+ pw.println(" -show_application_log_info Show the containerIds which");
+ pw.println(" belong to the specific");
+ pw.println(" Application. You can combine this");
+ pw.println(" with --nodeAddress to get");
+ pw.println(" containerIds for all the");
+ pw.println(" containers on the specific");
+ pw.println(" NodeManager.");
+ pw.println(" -show_container_log_info Show the container log metadata,");
+ pw.println(" including log-file names, the");
+ pw.println(" size of the log files. You can");
+ pw.println(" combine this with --containerId");
+ pw.println(" to get log metadata for the");
+ pw.println(" specific container, or with");
+ pw.println(" --nodeAddress to get log metadata");
+ pw.println(" for all the containers on the");
+ pw.println(" specific NodeManager.");
+ pw.println(" -size <size> Prints the log file's first 'n'");
+ pw.println(" bytes or the last 'n' bytes. Use");
+ pw.println(" negative values as bytes to read");
+ pw.println(" from the end and positive values");
+ pw.println(" as bytes to read from the");
+ pw.println(" beginning.");
pw.close();
String appReportStr = baos.toString("UTF-8");
Assert.assertEquals(appReportStr, sysOutStream.toString());
@@ -346,7 +359,7 @@ public class TestLogsCLI {
sysOutStream.reset();
exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
- "-log_files", ".*", "-regex"});
+ "-log_files_pattern", ".*"});
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
logMessage(containerId1, "syslog")));
@@ -392,7 +405,7 @@ public class TestLogsCLI {
sysOutStream.reset();
exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
- "-log_files", "std*", "-regex"});
+ "-log_files_pattern", "std*"});
assertTrue(exitCode == 0);
assertFalse(sysOutStream.toString().contains(
logMessage(containerId1, "syslog")));
@@ -809,6 +822,16 @@ public class TestLogsCLI {
+ "show_container_log_info."));
sysErrStream.reset();
+ // Specify log_files and log_files_pattern
+ // at the same time
+ exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
+ "-log_files", "*", "-log_files_pattern", ".*"});
+ assertTrue(exitCode == -1);
+ assertTrue(sysErrStream.toString().contains("Invalid options. "
+ + "Can only accept one of log_files/"
+ + "log_files_pattern."));
+ sysErrStream.reset();
+
// Specify a file name to the option -out
try {
fs.mkdirs(localPath);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org