You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2017/09/07 21:36:31 UTC
[18/37] hadoop git commit: YARN-7148. TestLogsCLI fails in trunk and
branch-2 and javadoc error. Contributed by Xuan Gong.
YARN-7148. TestLogsCLI fails in trunk and branch-2 and javadoc error. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/63231a2a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/63231a2a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/63231a2a
Branch: refs/heads/HDFS-7240
Commit: 63231a2a3008794c5ebcbc9d3855638fc5f28216
Parents: 1f3bc63
Author: Junping Du <ju...@apache.org>
Authored: Wed Sep 6 13:19:34 2017 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Wed Sep 6 13:19:34 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/client/cli/TestLogsCLI.java | 16 ++--
.../logaggregation/AggregatedLogFormat.java | 78 ++++++++++----------
.../logaggregation/LogAggregationUtils.java | 32 ++++----
.../yarn/logaggregation/LogCLIHelpers.java | 41 ++++++++--
.../LogAggregationFileController.java | 11 +--
.../tfile/LogAggregationTFileController.java | 1 -
6 files changed, 103 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63231a2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index 509a790..d58732e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -285,7 +285,7 @@ public class TestLogsCLI {
@Test (timeout = 15000)
public void testFetchFinishedApplictionLogs() throws Exception {
String remoteLogRootDir = "target/logs/";
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@@ -646,7 +646,7 @@ public class TestLogsCLI {
any(ContainerId.class));
// create local logs
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
FileSystem fs = FileSystem.get(configuration);
String rootLogDir = "target/LocalLogs";
Path rootLogDirPath = new Path(rootLogDir);
@@ -790,7 +790,7 @@ public class TestLogsCLI {
UserGroupInformation testUgi = UserGroupInformation
.createRemoteUser(testUser);
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@@ -921,7 +921,7 @@ public class TestLogsCLI {
public void testLogsCLIWithInvalidArgs() throws Exception {
String localDir = "target/SaveLogs";
Path localPath = new Path(localDir);
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
FileSystem fs = FileSystem.get(configuration);
ApplicationId appId = ApplicationId.newInstance(0, 1);
YarnClient mockYarnClient =
@@ -992,7 +992,7 @@ public class TestLogsCLI {
String localDir = "target/SaveLogs";
Path localPath = new Path(localDir);
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@@ -1083,7 +1083,7 @@ public class TestLogsCLI {
@Test (timeout = 15000)
public void testPrintContainerLogMetadata() throws Exception {
String remoteLogRootDir = "target/logs/";
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@@ -1188,7 +1188,7 @@ public class TestLogsCLI {
@Test (timeout = 15000)
public void testListNodeInfo() throws Exception {
String remoteLogRootDir = "target/logs/";
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
@@ -1239,7 +1239,7 @@ public class TestLogsCLI {
@Test (timeout = 15000)
public void testFetchApplictionLogsHar() throws Exception {
String remoteLogRootDir = "target/logs/";
- Configuration configuration = new Configuration();
+ Configuration configuration = new YarnConfiguration();
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
configuration
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63231a2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 3c1dcdc..9bec147 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -583,9 +583,9 @@ public class AggregatedLogFormat {
/**
* Returns the owner of the application.
- *
+ *
* @return the application owner.
- * @throws IOException
+ * @throws IOException if we can not get the application owner.
*/
public String getApplicationOwner() throws IOException {
TFile.Reader.Scanner ownerScanner = null;
@@ -610,9 +610,9 @@ public class AggregatedLogFormat {
/**
* Returns ACLs for the application. An empty map is returned if no ACLs are
* found.
- *
+ *
* @return a map of the Application ACLs.
- * @throws IOException
+ * @throws IOException if we can not get the application acls.
*/
public Map<ApplicationAccessType, String> getApplicationAcls()
throws IOException {
@@ -656,9 +656,10 @@ public class AggregatedLogFormat {
/**
* Read the next key and return the value-stream.
*
- * @param key
- * @return the valueStream if there are more keys or null otherwise.
- * @throws IOException
+ * @param key the log key
+ * @return the valueStream if there are more keys or null otherwise
+ * @throws IOException if we can not get the dataInputStream
+ * for the next key
*/
public DataInputStream next(LogKey key) throws IOException {
if (!this.atBeginning) {
@@ -683,10 +684,10 @@ public class AggregatedLogFormat {
* Get a ContainerLogsReader to read the logs for
* the specified container.
*
- * @param containerId
+ * @param containerId the containerId
* @return object to read the container's logs or null if the
* logs could not be found
- * @throws IOException
+ * @throws IOException if we can not get the container log reader.
*/
@Private
public ContainerLogsReader getContainerLogsReader(
@@ -752,10 +753,10 @@ public class AggregatedLogFormat {
/**
* Writes all logs for a single container to the provided writer.
- * @param valueStream
- * @param writer
- * @param logUploadedTime
- * @throws IOException
+ * @param valueStream the valueStream
+ * @param writer the log writer
+ * @param logUploadedTime the time stamp
+ * @throws IOException if we can not read the container logs.
*/
public static void readAcontainerLogs(DataInputStream valueStream,
Writer writer, long logUploadedTime) throws IOException {
@@ -780,9 +781,9 @@ public class AggregatedLogFormat {
/**
* Writes all logs for a single container to the provided writer.
- * @param valueStream
- * @param writer
- * @throws IOException
+ * @param valueStream the value stream
+ * @param writer the log writer
+ * @throws IOException if we can not read the container logs.
*/
public static void readAcontainerLogs(DataInputStream valueStream,
Writer writer) throws IOException {
@@ -849,10 +850,11 @@ public class AggregatedLogFormat {
* Keep calling this till you get a {@link EOFException} for getting logs of
* all types for a single container.
*
- * @param valueStream
- * @param out
- * @param logUploadedTime
- * @throws IOException
+ * @param valueStream the value stream
+ * @param out the print stream
+ * @param logUploadedTime the time stamp
+ * @throws IOException if we can not read the container log by specifying
+ * the container log type.
*/
public static void readAContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime)
@@ -864,11 +866,11 @@ public class AggregatedLogFormat {
* Keep calling this till you get a {@link EOFException} for getting logs of
* all types for a single container for the specific bytes.
*
- * @param valueStream
- * @param out
- * @param logUploadedTime
- * @param bytes
- * @throws IOException
+ * @param valueStream the value stream
+ * @param out the output print stream
+ * @param logUploadedTime the log upload time stamp
+ * @param bytes the output size of the log
+ * @throws IOException if we can not read the container log
*/
public static void readAContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime,
@@ -880,9 +882,9 @@ public class AggregatedLogFormat {
* Keep calling this till you get a {@link EOFException} for getting logs of
* all types for a single container.
*
- * @param valueStream
- * @param out
- * @throws IOException
+ * @param valueStream the value stream
+ * @param out the output print stream
+ * @throws IOException if we can not read the container log
*/
public static void readAContainerLogsForALogType(
DataInputStream valueStream, PrintStream out)
@@ -893,11 +895,11 @@ public class AggregatedLogFormat {
/**
* Keep calling this till you get a {@link EOFException} for getting logs of
* the specific types for a single container.
- * @param valueStream
- * @param out
- * @param logUploadedTime
- * @param logType
- * @throws IOException
+ * @param valueStream the value stream
+ * @param out the output print stream
+ * @param logUploadedTime the log uploaded time stamp
+ * @param logType the given log type
+ * @throws IOException if we can not read the container logs
*/
public static int readContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime,
@@ -909,11 +911,11 @@ public class AggregatedLogFormat {
/**
* Keep calling this till you get a {@link EOFException} for getting logs of
* the specific types for a single container.
- * @param valueStream
- * @param out
- * @param logUploadedTime
- * @param logType
- * @throws IOException
+ * @param valueStream the value stream
+ * @param out the output print stream
+ * @param logUploadedTime the log uploaded time stamp
+ * @param logType the given log type
+ * @throws IOException if we can not read the container logs
*/
public static int readContainerLogsForALogType(
DataInputStream valueStream, PrintStream out, long logUploadedTime,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63231a2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java
index e8a28de..6d04c29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java
@@ -38,11 +38,11 @@ public class LogAggregationUtils {
/**
* Constructs the full filename for an application's log file per node.
- * @param remoteRootLogDir
- * @param appId
- * @param user
- * @param nodeId
- * @param suffix
+ * @param remoteRootLogDir the aggregated remote root log dir
+ * @param appId the application Id
+ * @param user the application owner
+ * @param nodeId the node id
+ * @param suffix the log dir suffix
* @return the remote log file.
*/
public static Path getRemoteNodeLogFileForApp(Path remoteRootLogDir,
@@ -53,10 +53,10 @@ public class LogAggregationUtils {
/**
* Gets the remote app log dir.
- * @param remoteRootLogDir
- * @param appId
- * @param user
- * @param suffix
+ * @param remoteRootLogDir the aggregated log remote root log dir
+ * @param appId the application id
+ * @param user the application owner
+ * @param suffix the log directory suffix
* @return the remote application specific log dir.
*/
public static Path getRemoteAppLogDir(Path remoteRootLogDir,
@@ -67,9 +67,9 @@ public class LogAggregationUtils {
/**
* Gets the remote suffixed log dir for the user.
- * @param remoteRootLogDir
- * @param user
- * @param suffix
+ * @param remoteRootLogDir the aggregated log remote root log dir
+ * @param user the application owner
+ * @param suffix the log dir suffix
* @return the remote suffixed log dir.
*/
public static Path getRemoteLogSuffixedDir(Path remoteRootLogDir,
@@ -83,8 +83,8 @@ public class LogAggregationUtils {
/**
* Gets the remote log user dir.
- * @param remoteRootLogDir
- * @param user
+ * @param remoteRootLogDir the aggregated log remote root log dir
+ * @param user the application owner
* @return the remote per user log dir.
*/
public static Path getRemoteLogUserDir(Path remoteRootLogDir, String user) {
@@ -93,7 +93,7 @@ public class LogAggregationUtils {
/**
* Returns the suffix component of the log dir.
- * @param conf
+ * @param conf the configuration
* @return the suffix which will be appended to the user log dir.
*/
public static String getRemoteNodeLogDirSuffix(Configuration conf) {
@@ -104,7 +104,7 @@ public class LogAggregationUtils {
/**
* Converts a nodeId to a form used in the app log file name.
- * @param nodeId
+ * @param nodeId the nodeId
* @return the node string to be used to construct the file name.
*/
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63231a2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index 03acb33..0068eae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -128,8 +128,17 @@ public class LogCLIHelpers implements Configurable {
@VisibleForTesting
public int dumpAContainerLogsForLogType(ContainerLogsRequest options,
boolean outputFailure) throws IOException {
- boolean foundAnyLogs = this.getFileController(options.getAppId(),
- options.getAppOwner()).readAggregatedLogs(options, null);
+ LogAggregationFileController fc = null;
+ try {
+ fc = this.getFileController(
+ options.getAppId(), options.getAppOwner());
+ } catch (IOException ex) {
+ System.err.println(ex);
+ }
+ boolean foundAnyLogs = false;
+ if (fc != null) {
+ foundAnyLogs = fc.readAggregatedLogs(options, null);
+ }
if (!foundAnyLogs) {
if (outputFailure) {
containerLogNotFound(options.getContainerId());
@@ -142,9 +151,17 @@ public class LogCLIHelpers implements Configurable {
@Private
public int dumpAContainerLogsForLogTypeWithoutNodeId(
ContainerLogsRequest options) throws IOException {
- boolean foundAnyLogs = getFileController(options.getAppId(),
- options.getAppOwner()).readAggregatedLogs(
- options, null);
+ LogAggregationFileController fc = null;
+ try {
+ fc = this.getFileController(
+ options.getAppId(), options.getAppOwner());
+ } catch (IOException ex) {
+ System.err.println(ex);
+ }
+ boolean foundAnyLogs = false;
+ if (fc != null) {
+ foundAnyLogs = fc.readAggregatedLogs(options, null);
+ }
if (!foundAnyLogs) {
containerLogNotFound(options.getContainerId());
return -1;
@@ -155,9 +172,17 @@ public class LogCLIHelpers implements Configurable {
@Private
public int dumpAllContainersLogs(ContainerLogsRequest options)
throws IOException {
- boolean foundAnyLogs = getFileController(options.getAppId(),
- options.getAppOwner()).readAggregatedLogs(
- options, null);
+ LogAggregationFileController fc = null;
+ try {
+ fc = this.getFileController(
+ options.getAppId(), options.getAppOwner());
+ } catch (IOException ex) {
+ System.err.println(ex);
+ }
+ boolean foundAnyLogs = false;
+ if (fc != null) {
+ foundAnyLogs = fc.readAggregatedLogs(options, null);
+ }
if (!foundAnyLogs) {
emptyLogDir(LogAggregationUtils.getRemoteAppLogDir(
conf, options.getAppId(), options.getAppOwner())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63231a2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index 87344a7..aafdb66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -201,6 +201,7 @@ public abstract class LogAggregationFileController {
* Output container log.
* @param logRequest {@link ContainerLogsRequest}
* @param os the output stream
+ * @return true if we can read the aggregated logs successfully
* @throws IOException if we can not access the log file.
*/
public abstract boolean readAggregatedLogs(ContainerLogsRequest logRequest,
@@ -228,9 +229,9 @@ public abstract class LogAggregationFileController {
/**
* Returns the owner of the application.
*
- * @param the aggregatedLog path.
- * @return the application owner.
- * @throws IOException
+ * @param aggregatedLogPath the aggregatedLog path
+ * @return the application owner
+ * @throws IOException if we can not get the application owner
*/
public abstract String getApplicationOwner(Path aggregatedLogPath)
throws IOException;
@@ -239,9 +240,9 @@ public abstract class LogAggregationFileController {
* Returns ACLs for the application. An empty map is returned if no ACLs are
* found.
*
- * @param the aggregatedLog path.
+ * @param aggregatedLogPath the aggregatedLog path.
* @return a map of the Application ACLs.
- * @throws IOException
+ * @throws IOException if we can not get the application acls
*/
public abstract Map<ApplicationAccessType, String> getApplicationAcls(
Path aggregatedLogPath) throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63231a2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
index d2038e2..aaed538 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
@@ -172,7 +172,6 @@ public class LogAggregationTFileController
byte[] buf = new byte[65535];
while (nodeFiles != null && nodeFiles.hasNext()) {
final FileStatus thisNodeFile = nodeFiles.next();
- LOG.error(thisNodeFile.getPath().toString());
String nodeName = thisNodeFile.getPath().getName();
if (nodeName.equals(appId + ".har")) {
Path p = new Path("har:///"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org