You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/05/23 23:21:23 UTC
[45/50] hadoop git commit: YARN-5112. Excessive log warnings for
directory permission issue on NM recovery. Contributed by Jian He. (cherry
picked from commit 867cd2f5a2e5966d6a7c5f5accb2fce78f9e7778)
YARN-5112. Excessive log warnings for directory permission issue on NM recovery. Contributed by Jian He.
(cherry picked from commit 867cd2f5a2e5966d6a7c5f5accb2fce78f9e7778)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6161d9ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6161d9ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6161d9ba
Branch: refs/heads/HDFS-1312
Commit: 6161d9ba5230f553db5f5490dce67e2afd1e29ca
Parents: 3b3b630
Author: Junping Du <ju...@apache.org>
Authored: Sun May 22 16:02:57 2016 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Sun May 22 16:04:02 2016 -0700
----------------------------------------------------------------------
.../logaggregation/AppLogAggregatorImpl.java | 45 +------------------
.../logaggregation/LogAggregationService.java | 47 ++++++++++++++++++--
2 files changed, 45 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6161d9ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index 32b0934..ac43ecf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -95,11 +95,6 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
// NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS.
private static final String NM_LOG_AGGREGATION_DEBUG_ENABLED
= YarnConfiguration.NM_PREFIX + "log-aggregation.debug-enabled";
- private static final boolean
- DEFAULT_NM_LOG_AGGREGATION_DEBUG_ENABLED = false;
-
- private static final long
- NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS = 3600;
private final LocalDirsHandlerService dirsHandler;
private final Dispatcher dispatcher;
@@ -142,7 +137,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
LocalDirsHandlerService dirsHandler, Path remoteNodeLogFileForApp,
Map<ApplicationAccessType, String> appAcls,
LogAggregationContext logAggregationContext, Context context,
- FileContext lfs) {
+ FileContext lfs, long rollingMonitorInterval) {
this.dispatcher = dispatcher;
this.conf = conf;
this.delService = deletionService;
@@ -167,43 +162,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
} else {
this.retentionSize = configuredRentionSize;
}
- long configuredRollingMonitorInterval = conf.getLong(
- YarnConfiguration
- .NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS,
- YarnConfiguration
- .DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS);
- boolean debug_mode =
- conf.getBoolean(NM_LOG_AGGREGATION_DEBUG_ENABLED,
- DEFAULT_NM_LOG_AGGREGATION_DEBUG_ENABLED);
- if (configuredRollingMonitorInterval > 0
- && configuredRollingMonitorInterval <
- NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS) {
- if (debug_mode) {
- this.rollingMonitorInterval = configuredRollingMonitorInterval;
- } else {
- LOG.warn(
- "rollingMonitorIntervall should be more than or equal to "
- + NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS
- + " seconds. Using "
- + NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS
- + " seconds instead.");
- this.rollingMonitorInterval =
- NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS;
- }
- } else {
- if (configuredRollingMonitorInterval <= 0) {
- LOG.info("rollingMonitorInterval is set as "
- + configuredRollingMonitorInterval + ". "
- + "The log rolling monitoring interval is disabled. "
- + "The logs will be aggregated after this application is finished.");
- } else {
- LOG.info("rollingMonitorInterval is set as "
- + configuredRollingMonitorInterval + ". "
- + "The logs will be aggregated every "
- + configuredRollingMonitorInterval + " seconds");
- }
- this.rollingMonitorInterval = configuredRollingMonitorInterval;
- }
+ this.rollingMonitorInterval = rollingMonitorInterval;
this.logAggregationInRolling =
this.rollingMonitorInterval <= 0 || this.logAggregationContext == null
|| this.logAggregationContext.getRolledLogsIncludePattern() == null
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6161d9ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
index 189e7ff..afc75c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
@@ -69,8 +69,14 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class LogAggregationService extends AbstractService implements
LogHandler {
- private static final Log LOG = LogFactory
- .getLog(LogAggregationService.class);
+ private static final Log LOG = LogFactory.getLog(LogAggregationService.class);
+ private static final long MIN_LOG_ROLLING_INTERVAL = 3600;
+ // This configuration is for debug and test purpose. By setting
+ // this configuration as true. We can break the lower bound of
+ // NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS.
+ private static final String NM_LOG_AGGREGATION_DEBUG_ENABLED
+ = YarnConfiguration.NM_PREFIX + "log-aggregation.debug-enabled";
+ private long rollingMonitorInterval;
/*
* Expected deployment TLD will be 1777, owner=<NMOwner>, group=<NMGroup -
@@ -101,6 +107,7 @@ public class LogAggregationService extends AbstractService implements
private NodeId nodeId;
private final ConcurrentMap<ApplicationId, AppLogAggregator> appLogAggregators;
+ private boolean logPermError = true;
@VisibleForTesting
ExecutorService threadPool;
@@ -128,6 +135,35 @@ public class LogAggregationService extends AbstractService implements
new ThreadFactoryBuilder()
.setNameFormat("LogAggregationService #%d")
.build());
+
+ rollingMonitorInterval = conf.getLong(
+ YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS,
+ YarnConfiguration.DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS);
+
+ boolean logAggregationDebugMode =
+ conf.getBoolean(NM_LOG_AGGREGATION_DEBUG_ENABLED, false);
+
+ if (rollingMonitorInterval > 0
+ && rollingMonitorInterval < MIN_LOG_ROLLING_INTERVAL) {
+ if (logAggregationDebugMode) {
+ LOG.info("Log aggregation debug mode enabled. rollingMonitorInterval = "
+ + rollingMonitorInterval);
+ } else {
+ LOG.warn("rollingMonitorIntervall should be more than or equal to "
+ + MIN_LOG_ROLLING_INTERVAL + " seconds. Using "
+ + MIN_LOG_ROLLING_INTERVAL + " seconds instead.");
+ this.rollingMonitorInterval = MIN_LOG_ROLLING_INTERVAL;
+ }
+ } else if (rollingMonitorInterval <= 0) {
+ LOG.info("rollingMonitorInterval is set as " + rollingMonitorInterval
+ + ". The log rolling monitoring interval is disabled. "
+ + "The logs will be aggregated after this application is finished.");
+ } else {
+ LOG.info("rollingMonitorInterval is set as " + rollingMonitorInterval
+ + ". The logs will be aggregated every " + rollingMonitorInterval
+ + " seconds");
+ }
+
super.serviceInit(conf);
}
@@ -197,11 +233,14 @@ public class LogAggregationService extends AbstractService implements
try {
FsPermission perms =
remoteFS.getFileStatus(this.remoteRootLogDir).getPermission();
- if (!perms.equals(TLDIR_PERMISSIONS)) {
+ if (!perms.equals(TLDIR_PERMISSIONS) && logPermError) {
LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir
+ "] already exist, but with incorrect permissions. "
+ "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms
+ "]." + " The cluster may have problems with multiple users.");
+ logPermError = false;
+ } else {
+ logPermError = true;
}
} catch (FileNotFoundException e) {
remoteExists = false;
@@ -360,7 +399,7 @@ public class LogAggregationService extends AbstractService implements
getConfig(), appId, userUgi, this.nodeId, dirsHandler,
getRemoteNodeLogFileForApp(appId, user),
appAcls, logAggregationContext, this.context,
- getLocalFileContext(getConfig()));
+ getLocalFileContext(getConfig()), this.rollingMonitorInterval);
if (this.appLogAggregators.putIfAbsent(appId, appLogAggregator) != null) {
throw new YarnRuntimeException("Duplicate initApp for " + appId);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org