You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/07/06 22:57:57 UTC
[35/48] hadoop git commit: YARN-3793. Several NPEs when deleting
local files on NM recovery. Contributed by Varun Saxena
YARN-3793. Several NPEs when deleting local files on NM recovery. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e556ee03
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e556ee03
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e556ee03
Branch: refs/heads/YARN-2928
Commit: e556ee030ff3610c8c407caa18eddd2a6c443d9d
Parents: d7e6226
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jul 1 21:13:32 2015 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jul 6 11:32:00 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../logaggregation/AppLogAggregatorImpl.java | 6 ++--
.../TestLogAggregationService.java | 36 ++++++++++++++++++++
3 files changed, 42 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e556ee03/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fff46b8..c4c7fa9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -701,6 +701,9 @@ Release 2.7.2 - UNRELEASED
BUG FIXES
+ YARN-3793. Several NPEs when deleting local files on NM recovery (Varun
+ Saxena via jlowe)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e556ee03/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index 4b95a03..654eb0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -276,10 +276,10 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
aggregator.doContainerLogAggregation(writer, appFinished);
if (uploadedFilePathsInThisCycle.size() > 0) {
uploadedLogsInThisCycle = true;
+ this.delService.delete(this.userUgi.getShortUserName(), null,
+ uploadedFilePathsInThisCycle
+ .toArray(new Path[uploadedFilePathsInThisCycle.size()]));
}
- this.delService.delete(this.userUgi.getShortUserName(), null,
- uploadedFilePathsInThisCycle
- .toArray(new Path[uploadedFilePathsInThisCycle.size()]));
// This container is finished, and all its logs have been uploaded,
// remove it from containerLogAggregators.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e556ee03/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index fd97cef..6a3d270 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -31,6 +31,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -280,6 +281,41 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
verifyLocalFileDeletion(logAggregationService);
}
+ /* Test to verify fix for YARN-3793 */
+ @Test
+ public void testNoLogsUploadedOnAppFinish() throws Exception {
+ this.delSrvc = new DeletionService(createContainerExecutor());
+ delSrvc = spy(delSrvc);
+ this.delSrvc.init(conf);
+ this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
+ this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+ this.remoteRootLogDir.getAbsolutePath());
+
+ LogAggregationService logAggregationService = new LogAggregationService(
+ dispatcher, this.context, this.delSrvc, super.dirsHandler);
+ logAggregationService.init(this.conf);
+ logAggregationService.start();
+
+ ApplicationId app = BuilderUtils.newApplicationId(1234, 1);
+ File appLogDir = new File(localLogDir, ConverterUtils.toString(app));
+ appLogDir.mkdir();
+ LogAggregationContext context =
+ LogAggregationContext.newInstance("HOST*", "sys*");
+ logAggregationService.handle(new LogHandlerAppStartedEvent(app, this.user,
+ null, ContainerLogsRetentionPolicy.ALL_CONTAINERS, this.acls, context));
+
+ ApplicationAttemptId appAttemptId =
+ BuilderUtils.newApplicationAttemptId(app, 1);
+ ContainerId cont = BuilderUtils.newContainerId(appAttemptId, 1);
+ writeContainerLogs(appLogDir, cont, new String[] { "stdout",
+ "stderr", "syslog" });
+ logAggregationService.handle(new LogHandlerContainerFinishedEvent(cont, 0));
+ logAggregationService.handle(new LogHandlerAppFinishedEvent(app));
+ logAggregationService.stop();
+ delSrvc.stop();
+ // Aggregated logs should not be deleted if not uploaded.
+ verify(delSrvc, times(0)).delete(user, null);
+ }
@Test
public void testNoContainerOnNode() throws Exception {