You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2018/05/22 20:14:30 UTC
[02/50] [abbrv] hadoop git commit: YARN-8128. Documenting the
per-node per-app file limit in YARN log aggregation. Contributed by Xuan
Gong.
YARN-8128. Documenting the per-node per-app file limit in YARN log aggregation. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/328f0847
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/328f0847
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/328f0847
Branch: refs/heads/HDDS-48
Commit: 328f0847e3d98cad8c368d57499f31081c153237
Parents: a2cdffb
Author: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Authored: Thu May 17 12:47:10 2018 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu May 17 12:47:10 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 10 ++++++++++
.../LogAggregationFileController.java | 17 ++++++-----------
.../src/main/resources/yarn-default.xml | 9 +++++++++
3 files changed, 25 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/328f0847/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 5ba2e05..8e56cb8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1377,6 +1377,16 @@ public class YarnConfiguration extends Configuration {
NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds";
public static final long
DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1;
+
+ /**
+ * Define how many aggregated log files per application per NM we can have
+ * in remote file system.
+ */
+ public static final String NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP
+ = NM_PREFIX + "log-aggregation.num-log-files-per-app";
+ public static final int
+ DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP = 30;
+
/**
* Number of threads used in log cleanup. Only applicable if Log aggregation
* is disabled
http://git-wip-us.apache.org/repos/asf/hadoop/blob/328f0847/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index aeef574..5ac89e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -95,13 +95,6 @@ public abstract class LogAggregationFileController {
protected static final FsPermission APP_LOG_FILE_UMASK = FsPermission
.createImmutable((short) (0640 ^ 0777));
- // This is temporary solution. The configuration will be deleted once
- // we find a more scalable method to only write a single log file per LRS.
- private static final String NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP
- = YarnConfiguration.NM_PREFIX + "log-aggregation.num-log-files-per-app";
- private static final int
- DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP = 30;
-
// This is temporary solution. The configuration will be deleted once we have
// the FileSystem API to check whether append operation is supported or not.
public static final String LOG_AGGREGATION_FS_SUPPORT_APPEND
@@ -122,12 +115,14 @@ public abstract class LogAggregationFileController {
*/
public void initialize(Configuration conf, String controllerName) {
this.conf = conf;
- int configuredRentionSize =
- conf.getInt(NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP,
- DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP);
+ int configuredRentionSize = conf.getInt(
+ YarnConfiguration.NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP,
+ YarnConfiguration
+ .DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP);
if (configuredRentionSize <= 0) {
this.retentionSize =
- DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP;
+ YarnConfiguration
+ .DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP;
} else {
this.retentionSize = configuredRentionSize;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/328f0847/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e078206..156ca24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3036,6 +3036,15 @@
</property>
<property>
+ <description>Define how many aggregated log files per application per NM
+ we can have in remote file system. By default, the total number of
+ aggregated log files per application per NM is 30.
+ </description>
+ <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
+ <value>30</value>
+ </property>
+
+ <property>
<description>
Enable/disable intermediate-data encryption at YARN level. For now,
this only is used by the FileSystemRMStateStore to setup right
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org