You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2019/08/14 15:37:54 UTC
[hadoop] branch branch-3.2 updated: YARN-9676. Add DEBUG and TRACE level messages to AppLogAggregatorImpl… (#1261)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new d5446b3 YARN-9676. Add DEBUG and TRACE level messages to AppLogAggregatorImpl… (#1261)
d5446b3 is described below
commit d5446b3a23d822a77db0638844faa5032c12d608
Author: Adam Antal <ad...@cloudera.com>
AuthorDate: Wed Aug 14 17:35:16 2019 +0200
YARN-9676. Add DEBUG and TRACE level messages to AppLogAggregatorImpl… (#1261)
* YARN-9676. Add DEBUG and TRACE level messages to AppLogAggregatorImpl and connected classes
* Using {} placeholder, and increasing loglevel if log aggregation failed.
(cherry picked from commit c89bdfacc8715fa6d72acd85437ab8cd257c8aad)
---
.../logaggregation/AppLogAggregatorImpl.java | 42 +++++++++++++---------
1 file changed, 25 insertions(+), 17 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index 84eec4f..965734f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -108,7 +108,6 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
// These variables are only for testing
private final AtomicBoolean waiting = new AtomicBoolean(false);
private int logAggregationTimes = 0;
- private int cleanupOldLogTimes = 0;
private long logFileSizeThreshold;
private boolean renameTemporaryLogFileFailed = false;
@@ -196,10 +195,16 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
this.remoteNodeTmpLogFileForApp = getRemoteNodeTmpLogFileForApp();
}
boolean logAggregationInRolling =
- rollingMonitorInterval <= 0 || this.logAggregationContext == null
- || this.logAggregationContext.getRolledLogsIncludePattern() == null
- || this.logAggregationContext.getRolledLogsIncludePattern()
- .isEmpty() ? false : true;
+ rollingMonitorInterval > 0 && this.logAggregationContext != null
+ && this.logAggregationContext.getRolledLogsIncludePattern() != null
+ && !this.logAggregationContext.getRolledLogsIncludePattern()
+ .isEmpty();
+ if (logAggregationInRolling) {
+ LOG.info("Rolling mode is turned on with include pattern {}",
+ this.logAggregationContext.getRolledLogsIncludePattern());
+ } else {
+ LOG.debug("Rolling mode is turned off");
+ }
logControllerContext = new LogAggregationFileControllerContext(
this.remoteNodeLogFileForApp,
this.remoteNodeTmpLogFileForApp,
@@ -299,11 +304,13 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
}
if (pendingContainerInThisCycle.isEmpty()) {
+ LOG.debug("No pending container in this cycle");
sendLogAggregationReport(true, "", appFinished);
return;
}
logAggregationTimes++;
+ LOG.debug("Cycle #{} of log aggregator", logAggregationTimes);
String diagnosticMessage = "";
boolean logAggregationSucceedInThisCycle = true;
DeletionTask deletionTask = null;
@@ -331,6 +338,8 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
appFinished, finishedContainers.contains(container));
if (uploadedFilePathsInThisCycle.size() > 0) {
uploadedLogsInThisCycle = true;
+ LOG.trace("Uploaded the following files for {}: {}",
+ container, uploadedFilePathsInThisCycle.toString());
List<Path> uploadedFilePathsInThisCycleList = new ArrayList<>();
uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle);
if (LOG.isDebugEnabled()) {
@@ -386,6 +395,13 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
if (logAggregationSucceedInThisCycle && deletionTask != null) {
delService.delete(deletionTask);
}
+ if (!diagnosticMessage.isEmpty()) {
+ LOG.debug("Sending log aggregation report along with the " +
+ "following diagnostic message:\"{}\"", diagnosticMessage);
+ }
+ if (!logAggregationSucceedInThisCycle) {
+ LOG.warn("Log aggregation did not succeed in this cycle");
+ }
sendLogAggregationReport(logAggregationSucceedInThisCycle,
diagnosticMessage, appFinished);
if (exc != null) {
@@ -524,14 +540,17 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
lfs.getFileStatus(logPath);
localAppLogDirs.add(logPath);
} catch (UnsupportedFileSystemException ue) {
- LOG.warn("Log dir " + rootLogDir + "is an unsupported file system", ue);
+ LOG.warn("Log dir {} is in an unsupported file system", rootLogDir,
+ ue);
continue;
} catch (IOException fe) {
+ LOG.warn("An exception occurred while getting file information", fe);
continue;
}
}
if (localAppLogDirs.size() > 0) {
+ LOG.debug("Cleaning up {} files", localAppLogDirs.size());
List<Path> localAppLogDirsList = new ArrayList<>();
localAppLogDirsList.addAll(localAppLogDirs);
DeletionTask deletionTask = new FileDeletionTask(delService,
@@ -674,17 +693,6 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
return userUgi;
}
- @Private
- @VisibleForTesting
- public int getLogAggregationTimes() {
- return this.logAggregationTimes;
- }
-
- @VisibleForTesting
- int getCleanupOldLogTimes() {
- return this.cleanupOldLogTimes;
- }
-
@VisibleForTesting
public LogAggregationFileController getLogAggregationFileController() {
return this.logAggregationFileController;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org