You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2016/03/29 01:01:50 UTC
hadoop git commit: YARN-4773. Log aggregation performs extraneous
filesystem operations when rolling log aggregation is disabled. Contributed
by Jun Gong
Repository: hadoop
Updated Branches:
refs/heads/trunk 8bfaa8003 -> 948b75807
YARN-4773. Log aggregation performs extraneous filesystem operations when rolling log aggregation is disabled. Contributed by Jun Gong
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/948b7580
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/948b7580
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/948b7580
Branch: refs/heads/trunk
Commit: 948b75807068c304ffe789e32f2b850c0d653e0a
Parents: 8bfaa80
Author: Jason Lowe <jl...@apache.org>
Authored: Mon Mar 28 23:00:56 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Mon Mar 28 23:00:56 2016 +0000
----------------------------------------------------------------------
.../logaggregation/AppLogAggregatorImpl.java | 14 ++++++++++----
.../logaggregation/TestLogAggregationService.java | 12 +++++++-----
2 files changed, 17 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/948b7580/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index da7fc14..fed4a3b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -124,11 +124,11 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
private final long rollingMonitorInterval;
private final boolean logAggregationInRolling;
private final NodeId nodeId;
- // This variable is only for testing
- private final AtomicBoolean waiting = new AtomicBoolean(false);
- // This variable is only for testing
+ // These variables are only for testing
+ private final AtomicBoolean waiting = new AtomicBoolean(false);
private int logAggregationTimes = 0;
+ private int cleanupOldLogTimes = 0;
private boolean renameTemporaryLogFileFailed = false;
@@ -365,8 +365,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
// Before upload logs, make sure the number of existing logs
// is smaller than the configured NM log aggregation retention size.
- if (uploadedLogsInThisCycle) {
+ if (uploadedLogsInThisCycle && logAggregationInRolling) {
cleanOldLogs();
+ cleanupOldLogTimes++;
}
if (writer != null) {
@@ -689,4 +690,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
public int getLogAggregationTimes() {
return this.logAggregationTimes;
}
+
+ @VisibleForTesting
+ int getCleanupOldLogTimes() {
+ return this.cleanupOldLogTimes;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/948b7580/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index 0392b38..0445c79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -2276,7 +2276,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
Records.newRecord(LogAggregationContext.class);
logAggregationContext.setLogAggregationPolicyClassName(
FailedOrKilledContainerLogAggregationPolicy.class.getName());
- verifySkipUnnecessaryNNOperations(logAggregationContext, 0, 2);
+ verifySkipUnnecessaryNNOperations(logAggregationContext, 0, 2, 0);
}
@Test (timeout = 20000)
@@ -2290,13 +2290,13 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
AMOnlyLogAggregationPolicy.class.getName());
contextWithAMOnly.setRolledLogsIncludePattern("sys*");
contextWithAMOnly.setRolledLogsExcludePattern("std_final");
- verifySkipUnnecessaryNNOperations(contextWithAMOnly, 1, 4);
+ verifySkipUnnecessaryNNOperations(contextWithAMOnly, 1, 4, 1);
}
private void verifySkipUnnecessaryNNOperations(
LogAggregationContext logAggregationContext,
- int expectedLogAggregationTimes, int expectedAggregationReportNum)
- throws Exception {
+ int expectedLogAggregationTimes, int expectedAggregationReportNum,
+ int expectedCleanupOldLogsTimes) throws Exception {
LogAggregationService logAggregationService = new LogAggregationService(
dispatcher, this.context, this.delSrvc, super.dirsHandler);
logAggregationService.init(this.conf);
@@ -2307,7 +2307,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
null, this.acls, logAggregationContext));
// Container finishes
- String[] logFiles = new String[] { "stdout" };
+ String[] logFiles = new String[] { "sysout" };
finishContainer(appId, logAggregationService,
ContainerType.APPLICATION_MASTER, 1, 0, logFiles);
AppLogAggregatorImpl aggregator =
@@ -2327,6 +2327,8 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
aggregator.getLogAggregationTimes());
assertEquals(expectedAggregationReportNum,
this.context.getLogAggregationStatusForApps().size());
+ assertEquals(expectedCleanupOldLogsTimes,
+ aggregator.getCleanupOldLogTimes());
}
private int numOfLogsAvailable(LogAggregationService logAggregationService,