You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by da...@apache.org on 2012/07/17 21:46:45 UTC
svn commit: r1362621 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/
hadoop-yarn/h...
Author: daryn
Date: Tue Jul 17 19:46:44 2012
New Revision: 1362621
URL: http://svn.apache.org/viewvc?rev=1362621&view=rev
Log:
svn merge -c 1362618 FIXES: MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1362621&r1=1362620&r2=1362621&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue Jul 17 19:46:44 2012
@@ -590,6 +590,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4283. Display tail of aggregated logs by default (Jason Lowe via
bobby)
+ MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't
+ init. (Jason Lowe via daryn)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java?rev=1362621&r1=1362620&r2=1362621&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java Tue Jul 17 19:46:44 2012
@@ -342,14 +342,14 @@ public class LogAggregationService exten
// A container is complete. Put this containers' logs up for aggregation if
// this containers' logs are needed.
- if (!this.appLogAggregators.containsKey(
- containerId.getApplicationAttemptId().getApplicationId())) {
- throw new YarnException("Application is not initialized yet for "
- + containerId);
+ AppLogAggregator aggregator = this.appLogAggregators.get(
+ containerId.getApplicationAttemptId().getApplicationId());
+ if (aggregator == null) {
+ LOG.warn("Log aggregation is not initialized for " + containerId
+ + ", did it fail to start?");
+ return;
}
- this.appLogAggregators.get(
- containerId.getApplicationAttemptId().getApplicationId())
- .startContainerLogAggregation(containerId, exitCode == 0);
+ aggregator.startContainerLogAggregation(containerId, exitCode == 0);
}
private void stopApp(ApplicationId appId) {
@@ -357,11 +357,13 @@ public class LogAggregationService exten
// App is complete. Finish up any containers' pending log aggregation and
// close the application specific logFile.
- if (!this.appLogAggregators.containsKey(appId)) {
- throw new YarnException("Application is not initialized yet for "
- + appId);
+ AppLogAggregator aggregator = this.appLogAggregators.get(appId);
+ if (aggregator == null) {
+ LOG.warn("Log aggregation is not initialized for " + appId
+ + ", did it fail to start?");
+ return;
}
- this.appLogAggregators.get(appId).finishLogAggregation();
+ aggregator.finishLogAggregation();
}
@Override
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1362621&r1=1362620&r2=1362621&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Tue Jul 17 19:46:44 2012
@@ -380,7 +380,7 @@ public class TestLogAggregationService e
@Test
@SuppressWarnings("unchecked")
- public void testLogAggregationInitFailsWithoutKillingNM() throws Exception {
+ public void testLogAggregationFailsWithoutKillingNM() throws Exception {
this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
@@ -412,7 +412,16 @@ public class TestLogAggregationService e
new ApplicationFinishEvent(appId, "Application failed to init aggregation: KABOOM!")
};
checkEvents(appEventHandler, expectedEvents, false,
- "getType", "getApplicationID", "getDiagnostic");
+ "getType", "getApplicationID", "getDiagnostic");
+
+ // verify trying to collect logs for containers/apps we don't know about
+ // doesn't blow up and tear down the NM
+ logAggregationService.handle(new LogHandlerContainerFinishedEvent(
+ BuilderUtils.newContainerId(4, 1, 1, 1), 0));
+ dispatcher.await();
+ logAggregationService.handle(new LogHandlerAppFinishedEvent(
+ BuilderUtils.newApplicationId(1, 5)));
+ dispatcher.await();
}
private void writeContainerLogs(File appLogDir, ContainerId containerId)