You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/05/12 00:54:51 UTC
[17/50] [abbrv] hadoop git commit: YARN-3476. Nodemanager can fail to
delete local logs if log aggregation fails. Contributed by Rohith
YARN-3476. Nodemanager can fail to delete local logs if log aggregation fails. Contributed by Rohith
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25e2b021
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25e2b021
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25e2b021
Branch: refs/heads/HDFS-7240
Commit: 25e2b02122c4ed760227ab33c49d3445c23b9276
Parents: 30099a3
Author: Jason Lowe <jl...@apache.org>
Authored: Fri May 8 22:45:52 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri May 8 22:45:52 2015 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../logaggregation/AppLogAggregatorImpl.java | 19 ++++++++++++++-----
2 files changed, 17 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25e2b021/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6d547c1..83985b5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -433,6 +433,9 @@ Release 2.7.1 - UNRELEASED
YARN-3554. Default value for maximum nodemanager connect wait time is too
high (Naganarasimha G R via jlowe)
+ YARN-3476. Nodemanager can fail to delete local logs if log aggregation
+ fails (Rohith via jlowe)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25e2b021/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index 3f09e5d..3111f10 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -417,6 +417,11 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
public void run() {
try {
doAppLogAggregation();
+ } catch (Exception e) {
+ // do post clean up of log directories on any exception
+ LOG.error("Error occured while aggregating the log for the application "
+ + appId, e);
+ doAppLogAggregationPostCleanUp();
} finally {
if (!this.appAggregationFinished.get()) {
LOG.warn("Aggregation did not complete for application " + appId);
@@ -454,6 +459,15 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
// App is finished, upload the container logs.
uploadLogsForContainers(true);
+ doAppLogAggregationPostCleanUp();
+
+ this.dispatcher.getEventHandler().handle(
+ new ApplicationEvent(this.appId,
+ ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED));
+ this.appAggregationFinished.set(true);
+ }
+
+ private void doAppLogAggregationPostCleanUp() {
// Remove the local app-log-dirs
List<Path> localAppLogDirs = new ArrayList<Path>();
for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) {
@@ -474,11 +488,6 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
this.delService.delete(this.userUgi.getShortUserName(), null,
localAppLogDirs.toArray(new Path[localAppLogDirs.size()]));
}
-
- this.dispatcher.getEventHandler().handle(
- new ApplicationEvent(this.appId,
- ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED));
- this.appAggregationFinished.set(true);
}
private Path getRemoteNodeTmpLogFileForApp() {