You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2019/08/14 15:35:26 UTC
[hadoop] branch trunk updated: YARN-9676. Add DEBUG and TRACE level messages to AppLogAggregatorImpl… (#1261)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c89bdfa YARN-9676. Add DEBUG and TRACE level messages to AppLogAggregatorImpl… (#1261)
c89bdfa is described below
commit c89bdfacc8715fa6d72acd85437ab8cd257c8aad
Author: Adam Antal <ad...@cloudera.com>
AuthorDate: Wed Aug 14 17:35:16 2019 +0200
YARN-9676. Add DEBUG and TRACE level messages to AppLogAggregatorImpl… (#1261)
* YARN-9676. Add DEBUG and TRACE level messages to AppLogAggregatorImpl and connected classes
* Using {} placeholder, and increasing loglevel if log aggregation failed.
---
.../logaggregation/AppLogAggregatorImpl.java | 42 +++++++++++++---------
1 file changed, 25 insertions(+), 17 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index ef14d2a..77bec72 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -108,7 +108,6 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
// These variables are only for testing
private final AtomicBoolean waiting = new AtomicBoolean(false);
private int logAggregationTimes = 0;
- private int cleanupOldLogTimes = 0;
private long logFileSizeThreshold;
private boolean renameTemporaryLogFileFailed = false;
@@ -196,10 +195,16 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
this.remoteNodeTmpLogFileForApp = getRemoteNodeTmpLogFileForApp();
}
boolean logAggregationInRolling =
- rollingMonitorInterval <= 0 || this.logAggregationContext == null
- || this.logAggregationContext.getRolledLogsIncludePattern() == null
- || this.logAggregationContext.getRolledLogsIncludePattern()
- .isEmpty() ? false : true;
+ rollingMonitorInterval > 0 && this.logAggregationContext != null
+ && this.logAggregationContext.getRolledLogsIncludePattern() != null
+ && !this.logAggregationContext.getRolledLogsIncludePattern()
+ .isEmpty();
+ if (logAggregationInRolling) {
+ LOG.info("Rolling mode is turned on with include pattern {}",
+ this.logAggregationContext.getRolledLogsIncludePattern());
+ } else {
+ LOG.debug("Rolling mode is turned off");
+ }
logControllerContext = new LogAggregationFileControllerContext(
this.remoteNodeLogFileForApp,
this.remoteNodeTmpLogFileForApp,
@@ -299,11 +304,13 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
}
if (pendingContainerInThisCycle.isEmpty()) {
+ LOG.debug("No pending container in this cycle");
sendLogAggregationReport(true, "", appFinished);
return;
}
logAggregationTimes++;
+ LOG.debug("Cycle #{} of log aggregator", logAggregationTimes);
String diagnosticMessage = "";
boolean logAggregationSucceedInThisCycle = true;
DeletionTask deletionTask = null;
@@ -331,6 +338,8 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
appFinished, finishedContainers.contains(container));
if (uploadedFilePathsInThisCycle.size() > 0) {
uploadedLogsInThisCycle = true;
+ LOG.trace("Uploaded the following files for {}: {}",
+ container, uploadedFilePathsInThisCycle.toString());
List<Path> uploadedFilePathsInThisCycleList = new ArrayList<>();
uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle);
if (LOG.isDebugEnabled()) {
@@ -386,6 +395,13 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
if (logAggregationSucceedInThisCycle && deletionTask != null) {
delService.delete(deletionTask);
}
+ if (!diagnosticMessage.isEmpty()) {
+ LOG.debug("Sending log aggregation report along with the " +
+ "following diagnostic message:\"{}\"", diagnosticMessage);
+ }
+ if (!logAggregationSucceedInThisCycle) {
+ LOG.warn("Log aggregation did not succeed in this cycle");
+ }
sendLogAggregationReport(logAggregationSucceedInThisCycle,
diagnosticMessage, appFinished);
if (exc != null) {
@@ -522,14 +538,17 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
lfs.getFileStatus(logPath);
localAppLogDirs.add(logPath);
} catch (UnsupportedFileSystemException ue) {
- LOG.warn("Log dir " + rootLogDir + "is an unsupported file system", ue);
+ LOG.warn("Log dir {} is in an unsupported file system", rootLogDir,
+ ue);
continue;
} catch (IOException fe) {
+ LOG.warn("An exception occurred while getting file information", fe);
continue;
}
}
if (localAppLogDirs.size() > 0) {
+ LOG.debug("Cleaning up {} files", localAppLogDirs.size());
List<Path> localAppLogDirsList = new ArrayList<>();
localAppLogDirsList.addAll(localAppLogDirs);
DeletionTask deletionTask = new FileDeletionTask(delService,
@@ -672,17 +691,6 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
return userUgi;
}
- @Private
- @VisibleForTesting
- public int getLogAggregationTimes() {
- return this.logAggregationTimes;
- }
-
- @VisibleForTesting
- int getCleanupOldLogTimes() {
- return this.cleanupOldLogTimes;
- }
-
@VisibleForTesting
public LogAggregationFileController getLogAggregationFileController() {
return this.logAggregationFileController;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org