You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/08/26 00:44:49 UTC
[37/50] [abbrv] hadoop git commit: YARN-8649. NPE in localizer
hearbeat processing if a container is killed while localizing. Contributed by
lujie
YARN-8649. NPE in localizer hearbeat processing if a container is killed while localizing. Contributed by lujie
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/585ebd87
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/585ebd87
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/585ebd87
Branch: refs/heads/HDFS-12943
Commit: 585ebd873a55bedd2a364d256837f08ada8ba032
Parents: bed8cb6
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Aug 23 09:29:46 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Aug 23 09:29:46 2018 -0500
----------------------------------------------------------------------
.../localizer/LocalResourcesTrackerImpl.java | 5 +++++
.../localizer/ResourceLocalizationService.java | 12 ++++++++----
.../localizer/TestResourceLocalizationService.java | 12 +++++++++++-
3 files changed, 24 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585ebd87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
index dd31543..ad24c62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
@@ -500,6 +500,11 @@ class LocalResourcesTrackerImpl implements LocalResourcesTracker {
Path localPath = new Path(rPath, req.getPath().getName());
LocalizedResource rsrc = localrsrc.get(req);
+ if (rsrc == null) {
+ LOG.warn("Resource " + req + " has been removed"
+ + " and will no longer be localized");
+ return null;
+ }
rsrc.setLocalPath(localPath);
LocalResource lr = LocalResource.newInstance(req.getResource(),
req.getType(), req.getVisibility(), req.getSize(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585ebd87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 2892d1f..d9b887f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -885,6 +885,9 @@ public class ResourceLocalizationService extends CompositeService
Path publicDirDestPath =
publicRsrc.getPathForLocalization(key, publicRootPath,
delService);
+ if (publicDirDestPath == null) {
+ return;
+ }
if (!publicDirDestPath.getParent().equals(publicRootPath)) {
createParentDirs(publicDirDestPath, publicRootPath);
if (diskValidator != null) {
@@ -1175,10 +1178,11 @@ public class ResourceLocalizationService extends CompositeService
LocalResourcesTracker tracker = getLocalResourcesTracker(
next.getVisibility(), user, applicationId);
if (tracker != null) {
- ResourceLocalizationSpec resource =
- NodeManagerBuilderUtils.newResourceLocalizationSpec(next,
- getPathForLocalization(next, tracker));
- rsrcs.add(resource);
+ Path localPath = getPathForLocalization(next, tracker);
+ if (localPath != null) {
+ rsrcs.add(NodeManagerBuilderUtils.newResourceLocalizationSpec(
+ next, localPath));
+ }
}
} catch (IOException e) {
LOG.error("local path for PRIVATE localization could not be " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/585ebd87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index 2b9148e..21896ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -1717,8 +1717,18 @@ public class TestResourceLocalizationService {
assertEquals("NM should tell localizer to be LIVE in Heartbeat.",
LocalizerAction.LIVE, response.getLocalizerAction());
- // Cleanup application.
+ // Cleanup container.
spyService.handle(new ContainerLocalizationCleanupEvent(c, rsrcs));
+ dispatcher.await();
+ try {
+ /*Directly send heartbeat to introduce race as container
+ is being cleaned up.*/
+ locRunnerForContainer.processHeartbeat(
+ Collections.singletonList(rsrcSuccess));
+ } catch (Exception e) {
+ fail("Exception should not have been thrown on processing heartbeat");
+ }
+ // Cleanup application.
spyService.handle(new ApplicationLocalizationEvent(
LocalizationEventType.DESTROY_APPLICATION_RESOURCES, app));
dispatcher.await();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org