You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ju...@apache.org on 2016/01/06 23:07:47 UTC
hadoop git commit: YARN-4354. Public resource localization fails with
NPE. Contributed by Jason Lowe. (cherry picked from commit
855d52927b6115e2cfbd97a94d6c1a3ddf0e94bb) (cherry picked from commit
1d107d805841e9b39db01aa236068a0ff1ca46d1)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 9da7b1fdd -> cfbb03542
YARN-4354. Public resource localization fails with NPE. Contributed by Jason Lowe.
(cherry picked from commit 855d52927b6115e2cfbd97a94d6c1a3ddf0e94bb)
(cherry picked from commit 1d107d805841e9b39db01aa236068a0ff1ca46d1)
Conflicts:
hadoop-yarn-project/CHANGES.txt
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cfbb0354
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cfbb0354
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cfbb0354
Branch: refs/heads/branch-2.6
Commit: cfbb0354203f885f5c408123bbcd5e9039db18bb
Parents: 9da7b1f
Author: Junping Du <ju...@apache.org>
Authored: Sun Nov 15 04:43:57 2015 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Wed Jan 6 14:15:55 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../localizer/LocalResourcesTrackerImpl.java | 10 +++-
.../TestLocalResourcesTrackerImpl.java | 56 ++++++++++++++++++--
.../TestResourceLocalizationService.java | 4 +-
4 files changed, 67 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfbb0354/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0cf59cb..684476c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -34,6 +34,9 @@ Release 2.6.4 - UNRELEASED
YARN-2902. Killing a container that is localizing can orphan resources in
the DOWNLOADING state (Varun Saxena via jlowe)
+ YARN-4354. Public resource localization fails with NPE. (Jason Lowe via
+ junping_du)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfbb0354/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
index f9346d8..60a9529 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
@@ -171,14 +171,22 @@ class LocalResourcesTrackerImpl implements LocalResourcesTracker {
break;
}
+ if (rsrc == null) {
+ LOG.warn("Received " + event.getType() + " event for request " + req
+ + " but localized resource is missing");
+ return;
+ }
rsrc.handle(event);
// Remove the resource if its downloading and its reference count has
// become 0 after RELEASE. This maybe because a container was killed while
// localizing and no other container is referring to the resource.
+ // NOTE: This should NOT be done for public resources since the
+ // download is not associated with a container-specific localizer.
if (event.getType() == ResourceEventType.RELEASE) {
if (rsrc.getState() == ResourceState.DOWNLOADING &&
- rsrc.getRefCount() <= 0) {
+ rsrc.getRefCount() <= 0 &&
+ rsrc.getRequest().getVisibility() != LocalResourceVisibility.PUBLIC) {
removeResource(req);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfbb0354/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
index e59a796..8cf3b2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
@@ -137,12 +137,12 @@ public class TestLocalResourcesTrackerImpl {
tracker.handle(rel21Event);
dispatcher.await();
- verifyTrackedResourceCount(tracker, 1);
+ verifyTrackedResourceCount(tracker, 2);
// Verify resource with non zero ref count is not removed.
Assert.assertEquals(2, lr1.getRefCount());
Assert.assertFalse(tracker.remove(lr1, mockDelService));
- verifyTrackedResourceCount(tracker, 1);
+ verifyTrackedResourceCount(tracker, 2);
// Localize resource1
ResourceLocalizedEvent rle =
@@ -157,7 +157,7 @@ public class TestLocalResourcesTrackerImpl {
// Verify resources in state LOCALIZED with ref-count=0 is removed.
Assert.assertTrue(tracker.remove(lr1, mockDelService));
- verifyTrackedResourceCount(tracker, 0);
+ verifyTrackedResourceCount(tracker, 1);
} finally {
if (dispatcher != null) {
dispatcher.stop();
@@ -828,6 +828,56 @@ public class TestLocalResourcesTrackerImpl {
}
}
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testReleaseWhileDownloading() throws Exception {
+ String user = "testuser";
+ DrainDispatcher dispatcher = null;
+ try {
+ Configuration conf = new Configuration();
+ dispatcher = createDispatcher(conf);
+ EventHandler<LocalizerEvent> localizerEventHandler =
+ mock(EventHandler.class);
+ EventHandler<LocalizerEvent> containerEventHandler =
+ mock(EventHandler.class);
+ dispatcher.register(LocalizerEventType.class, localizerEventHandler);
+ dispatcher.register(ContainerEventType.class, containerEventHandler);
+
+ ContainerId cId = BuilderUtils.newContainerId(1, 1, 1, 1);
+ LocalizerContext lc = new LocalizerContext(user, cId, null);
+
+ LocalResourceRequest req =
+ createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
+ LocalizedResource lr = createLocalizedResource(req, dispatcher);
+ ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
+ new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
+ localrsrc.put(req, lr);
+ LocalResourcesTracker tracker =
+ new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc,
+ false, conf, new NMNullStateStoreService());
+
+ // request the resource
+ ResourceEvent reqEvent =
+ new ResourceRequestEvent(req, LocalResourceVisibility.PUBLIC, lc);
+ tracker.handle(reqEvent);
+
+ // release the resource
+ ResourceEvent relEvent = new ResourceReleaseEvent(req, cId);
+ tracker.handle(relEvent);
+
+ // download completing after release
+ ResourceLocalizedEvent rle =
+ new ResourceLocalizedEvent(req, new Path("file:///tmp/r1"), 1);
+ tracker.handle(rle);
+
+ dispatcher.await();
+ } finally {
+ if (dispatcher != null) {
+ dispatcher.stop();
+ }
+ }
+ }
+
private boolean createdummylocalizefile(Path path) {
boolean ret = false;
File file = new File(path.toUri().getRawPath().toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfbb0354/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index fdf9981..d19315d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -478,8 +478,8 @@ public class TestResourceLocalizationService {
Assert.assertEquals("Incorrect reference count", 0, lr.getRefCount());
pubRsrcs.remove(lr.getRequest());
}
- Assert.assertEquals(2, pubRsrcs.size());
- Assert.assertEquals(0, pubRsrcCount);
+ Assert.assertEquals(0, pubRsrcs.size());
+ Assert.assertEquals(2, pubRsrcCount);
appRsrcCount = 0;
for (LocalizedResource lr : appTracker) {