You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/09/01 00:41:21 UTC
hadoop git commit: YARN-3011. Possible IllegalArgumentException in
ResourceLocalizationService might lead NM to crash. Contributed by Varun
Saxena (cherry picked from commit 4e15fc08411318e11152fcd5a4648ed1d6fbb480)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6.1 24da94424 -> 994c3d049
YARN-3011. Possible IllegalArgumentException in ResourceLocalizationService might lead NM to crash. Contributed by Varun Saxena
(cherry picked from commit 4e15fc08411318e11152fcd5a4648ed1d6fbb480)
(cherry picked from commit 8100c8a68c32978a177af9a3e6639f6de533886d)
(cherry picked from commit 10a6c4f349e6f32ed2a520bf669a0cbfff31c824)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/994c3d04
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/994c3d04
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/994c3d04
Branch: refs/heads/branch-2.6.1
Commit: 994c3d049ab6fa215940a04bee34f9aa66b343f4
Parents: 24da944
Author: Jian He <ji...@apache.org>
Authored: Tue Jan 27 13:31:22 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Mon Aug 31 15:38:45 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../localizer/ResourceLocalizationService.java | 10 +++++++++
.../TestResourceLocalizationService.java | 23 ++++++++++++++++++--
3 files changed, 34 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/994c3d04/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3a8911e..828d3e9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -66,6 +66,9 @@ Release 2.6.1 - UNRELEASED
YARN-2997. Fixed NodeStatusUpdater to not send alreay-sent completed
container statuses on heartbeat. (Chengbing Liu via jianhe)
+ YARN-3011. Possible IllegalArgumentException in ResourceLocalizationService
+ might lead NM to crash. (Varun Saxena via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/994c3d04/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index f4b6221..bcda3c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -788,6 +788,13 @@ public class ResourceLocalizationService extends CompositeService
.getResource().getRequest(), e.getMessage()));
LOG.error("Local path for public localization is not found. "
+ " May be disks failed.", e);
+ } catch (IllegalArgumentException ie) {
+ rsrc.unlock();
+ publicRsrc.handle(new ResourceFailedLocalizationEvent(request
+ .getResource().getRequest(), ie.getMessage()));
+ LOG.error("Local path for public localization is not found. "
+ + " Incorrect path. " + request.getResource().getRequest()
+ .getPath(), ie);
} catch (RejectedExecutionException re) {
rsrc.unlock();
publicRsrc.handle(new ResourceFailedLocalizationEvent(request
@@ -1010,6 +1017,9 @@ public class ResourceLocalizationService extends CompositeService
} catch (IOException e) {
LOG.error("local path for PRIVATE localization could not be " +
"found. Disks might have failed.", e);
+ } catch (IllegalArgumentException e) {
+ LOG.error("Inorrect path for PRIVATE localization."
+ + next.getResource().getFile(), e);
} catch (URISyntaxException e) {
//TODO fail? Already translated several times...
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/994c3d04/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index bf36651..c549fce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -1126,14 +1126,33 @@ public class TestResourceLocalizationService {
user, appId);
Assert.assertNull(tracker.getLocalizedResource(pubReq));
- // test RejectedExecutionException
+ // test IllegalArgumentException
+ String name = Long.toHexString(r.nextLong());
+ URL url = getPath("/local/PRIVATE/" + name + "/");
+ final LocalResource rsrc =
+ BuilderUtils.newLocalResource(url, LocalResourceType.FILE,
+ LocalResourceVisibility.PUBLIC, r.nextInt(1024) + 1024L,
+ r.nextInt(1024) + 2048L);
+ final LocalResourceRequest pubReq1 = new LocalResourceRequest(rsrc);
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req1 =
+ new HashMap<LocalResourceVisibility,
+ Collection<LocalResourceRequest>>();
+ req1.put(LocalResourceVisibility.PUBLIC,
+ Collections.singletonList(pubReq1));
Mockito
.doCallRealMethod()
.when(dirsHandlerSpy)
.getLocalPathForWrite(isA(String.class), Mockito.anyLong(),
Mockito.anyBoolean());
+ // send request
+ spyService.handle(new ContainerLocalizationRequestEvent(c, req1));
+ dispatcher.await();
+ tracker =
+ spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC,
+ user, appId);
+ Assert.assertNull(tracker.getLocalizedResource(pubReq));
- // shutdown the thread pool
+ // test RejectedExecutionException by shutting down the thread pool
PublicLocalizer publicLocalizer = spyService.getPublicLocalizer();
publicLocalizer.threadPool.shutdown();