You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2019/08/09 11:24:00 UTC
[hadoop] branch branch-3.1 updated: Logging fileSize of log files
under NM Local Dir. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new be9ac8a Logging fileSize of log files under NM Local Dir. Contributed by Prabhu Joseph
be9ac8a is described below
commit be9ac8adf9d0e2327bc5d0871bd153bef9ec7694
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Fri Aug 2 13:38:06 2019 +0200
Logging fileSize of log files under NM Local Dir. Contributed by Prabhu Joseph
(cherry picked from commit 54ac80176e8487b7a18cd9e16a11efa289d0b7df)
---
.../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 5 +++++
.../src/main/resources/yarn-default.xml | 8 ++++++++
.../logaggregation/AppLogAggregatorImpl.java | 18 +++++++++++++++++-
3 files changed, 30 insertions(+), 1 deletion(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 9f3136d..abf6fcf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1322,6 +1322,11 @@ public class YarnConfiguration extends Configuration {
public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX
+ "log-aggregation.retain-seconds";
public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1;
+
+ public static final String LOG_AGGREGATION_DEBUG_FILESIZE = YARN_PREFIX
+ + "log-aggregation.debug.filesize";
+ public static final long DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE
+ = 100 * 1024 * 1024;
/**
* How long to wait between aggregated log retention checks. If set to
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index ebeecdb..dfb6cbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1263,6 +1263,14 @@
</property>
<property>
+ <description>The log files created under NM Local Directories
+ will be logged if it exceeds the configured bytes. This
+ only takes effect if log4j level is at least Debug.</description>
+ <name>yarn.log-aggregation.debug.filesize</name>
+ <value>104857600</value>
+ </property>
+
+ <property>
<description>Specify which log file controllers we will support. The first
file controller we add will be used to write the aggregated logs.
This comma separated configuration will work with the configuration:
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index c290eb4..5dfe6d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -108,7 +108,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
private final AtomicBoolean waiting = new AtomicBoolean(false);
private int logAggregationTimes = 0;
private int cleanupOldLogTimes = 0;
-
+ private long logFileSizeThreshold;
private boolean renameTemporaryLogFileFailed = false;
private final Map<ContainerId, ContainerLogAggregator> containerLogAggregators =
@@ -175,6 +175,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
this.nodeId = nodeId;
this.logAggPolicy = getLogAggPolicy(conf);
this.recoveredLogInitedTime = recoveredLogInitedTime;
+ this.logFileSizeThreshold =
+ conf.getLong(YarnConfiguration.LOG_AGGREGATION_DEBUG_FILESIZE,
+ YarnConfiguration.DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE);
if (logAggregationFileController == null) {
// by default, use T-File Controller
this.logAggregationFileController = new LogAggregationTFileController();
@@ -327,6 +330,19 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
uploadedLogsInThisCycle = true;
List<Path> uploadedFilePathsInThisCycleList = new ArrayList<>();
uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle);
+ if (LOG.isDebugEnabled()) {
+ for (Path uploadedFilePath : uploadedFilePathsInThisCycleList) {
+ try {
+ long fileSize = lfs.getFileStatus(uploadedFilePath).getLen();
+ if (fileSize >= logFileSizeThreshold) {
+ LOG.debug("Log File " + uploadedFilePath
+ + " size is " + fileSize + " bytes");
+ }
+ } catch (Exception e1) {
+ LOG.error("Failed to get log file size " + e1);
+ }
+ }
+ }
DeletionTask deletionTask = new FileDeletionTask(delService,
this.userUgi.getShortUserName(), null,
uploadedFilePathsInThisCycleList);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org