You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xg...@apache.org on 2017/06/05 20:30:56 UTC
[07/50] [abbrv] hadoop git commit: YARN-6641. Non-public resource
localization on a bad disk causes subsequent containers failure. Contributed
by Kuhu Shukla
YARN-6641. Non-public resource localization on a bad disk causes subsequent containers failure. Contributed by Kuhu Shukla
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47710c22
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47710c22
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47710c22
Branch: refs/heads/YARN-5734
Commit: 47710c22fc3a70e3fc1504f4235bd8ca5e1b1671
Parents: 65cbe43
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Fri May 26 09:37:56 2017 -0500
Committer: Xuan <xg...@apache.org>
Committed: Mon Jun 5 13:29:22 2017 -0700
----------------------------------------------------------------------
.../localizer/LocalResourcesTrackerImpl.java | 13 ++--
.../localizer/ResourceLocalizationService.java | 10 +--
.../TestLocalResourcesTrackerImpl.java | 8 +--
.../TestResourceLocalizationService.java | 71 ++++++++++++++++++++
4 files changed, 86 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47710c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
index 940c599..af34e92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
@@ -94,14 +94,6 @@ class LocalResourcesTrackerImpl implements LocalResourcesTracker {
public LocalResourcesTrackerImpl(String user, ApplicationId appId,
Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
- Configuration conf, NMStateStoreService stateStore) {
- this(user, appId, dispatcher,
- new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>(),
- useLocalCacheDirectoryManager, conf, stateStore, null);
- }
-
- public LocalResourcesTrackerImpl(String user, ApplicationId appId,
- Dispatcher dispatcher, boolean useLocalCacheDirectoryManager,
Configuration conf, NMStateStoreService stateStore,
LocalDirsHandlerService dirHandler) {
this(user, appId, dispatcher,
@@ -528,4 +520,9 @@ class LocalResourcesTrackerImpl implements LocalResourcesTracker {
}
return mgr;
}
+
+ @VisibleForTesting
+ LocalDirsHandlerService getDirsHandler() {
+ return dirsHandler;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47710c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 28fb53c..663bad7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -306,7 +306,7 @@ public class ResourceLocalizationService extends CompositeService
trackerState = userResources.getPrivateTrackerState();
if (!trackerState.isEmpty()) {
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
- null, dispatcher, true, super.getConfig(), stateStore);
+ null, dispatcher, true, super.getConfig(), stateStore, dirsHandler);
LocalResourcesTracker oldTracker = privateRsrc.putIfAbsent(user,
tracker);
if (oldTracker != null) {
@@ -322,7 +322,8 @@ public class ResourceLocalizationService extends CompositeService
ApplicationId appId = appEntry.getKey();
String appIdStr = appId.toString();
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
- appId, dispatcher, false, super.getConfig(), stateStore);
+ appId, dispatcher, false, super.getConfig(), stateStore,
+ dirsHandler);
LocalResourcesTracker oldTracker = appRsrc.putIfAbsent(appIdStr,
tracker);
if (oldTracker != null) {
@@ -460,10 +461,11 @@ public class ResourceLocalizationService extends CompositeService
// 0) Create application tracking structs
String userName = app.getUser();
privateRsrc.putIfAbsent(userName, new LocalResourcesTrackerImpl(userName,
- null, dispatcher, true, super.getConfig(), stateStore));
+ null, dispatcher, true, super.getConfig(), stateStore, dirsHandler));
String appIdStr = app.getAppId().toString();
appRsrc.putIfAbsent(appIdStr, new LocalResourcesTrackerImpl(app.getUser(),
- app.getAppId(), dispatcher, false, super.getConfig(), stateStore));
+ app.getAppId(), dispatcher, false, super.getConfig(), stateStore,
+ dirsHandler));
// 1) Signal container init
//
// This is handled by the ApplicationImpl state machine and allows
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47710c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
index 8d9705b..2874acb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
@@ -529,7 +529,7 @@ public class TestLocalResourcesTrackerImpl {
try {
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
- appId, dispatcher, false, conf, stateStore);
+ appId, dispatcher, false, conf, stateStore, null);
// Container 1 needs lr1 resource
ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
@@ -610,7 +610,7 @@ public class TestLocalResourcesTrackerImpl {
try {
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
- appId, dispatcher, false, conf, stateStore);
+ appId, dispatcher, false, conf, stateStore, null);
// Container 1 needs lr1 resource
ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
@@ -672,7 +672,7 @@ public class TestLocalResourcesTrackerImpl {
try {
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
- appId, dispatcher, false, conf, stateStore);
+ appId, dispatcher, false, conf, stateStore, null);
// Container 1 needs lr1 resource
ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
@@ -725,7 +725,7 @@ public class TestLocalResourcesTrackerImpl {
try {
LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user,
- appId, dispatcher, true, conf, stateStore);
+ appId, dispatcher, true, conf, stateStore, null);
LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
LocalResourceVisibility.PUBLIC);
Assert.assertNull(tracker.getLocalizedResource(lr1));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47710c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index 14eea9f..89cbeb4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -2820,4 +2820,75 @@ public class TestResourceLocalizationService {
}
}
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testDirHandler() throws Exception {
+ File f = new File(basedir.toString());
+ String[] sDirs = new String[4];
+ List<Path> localDirs = new ArrayList<Path>(sDirs.length);
+ for (int i = 0; i < 4; ++i) {
+ sDirs[i] = f.getAbsolutePath() + i;
+ localDirs.add(new Path(sDirs[i]));
+ }
+ conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
+ LocalizerTracker mockLocalizerTracker = mock(LocalizerTracker.class);
+ DrainDispatcher dispatcher = new DrainDispatcher();
+ dispatcher.init(conf);
+ dispatcher.start();
+ EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
+ dispatcher.register(ApplicationEventType.class, applicationBus);
+ EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class);
+ dispatcher.register(LocalizerEventType.class, localizerBus);
+
+ ContainerExecutor exec = mock(ContainerExecutor.class);
+ LocalDirsHandlerService mockDirsHandler =
+ mock(LocalDirsHandlerService.class);
+ doReturn(new ArrayList<String>(Arrays.asList(sDirs))).when(
+ mockDirsHandler).getLocalDirsForCleanup();
+ // setup mocks
+ DeletionService delService = mock(DeletionService.class);
+ ResourceLocalizationService rawService =
+ new ResourceLocalizationService(dispatcher, exec, delService,
+ mockDirsHandler, nmContext, metrics);
+ ResourceLocalizationService spyService = spy(rawService);
+ doReturn(mockServer).when(spyService).createServer();
+ doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker(
+ isA(Configuration.class));
+
+ final String user = "user0";
+ // init application
+ final Application app = mock(Application.class);
+ final ApplicationId appId =
+ BuilderUtils.newApplicationId(314159265358979L, 3);
+ when(app.getUser()).thenReturn(user);
+ when(app.getAppId()).thenReturn(appId);
+ when(app.toString()).thenReturn(appId.toString());
+ try {
+ spyService.init(conf);
+ spyService.start();
+
+ spyService.handle(new ApplicationLocalizationEvent(
+ LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
+ dispatcher.await();
+
+ LocalResourcesTracker appTracker =
+ spyService.getLocalResourcesTracker(
+ LocalResourceVisibility.APPLICATION, user, appId);
+ LocalResourcesTracker privTracker =
+ spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE,
+ user, appId);
+ LocalResourcesTracker pubTracker =
+ spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC,
+ user, appId);
+ Assert.assertNotNull("dirHandler for appTracker is null!",
+ ((LocalResourcesTrackerImpl)appTracker).getDirsHandler());
+ Assert.assertNotNull("dirHandler for privTracker is null!",
+ ((LocalResourcesTrackerImpl)privTracker).getDirsHandler());
+ Assert.assertNotNull("dirHandler for pubTracker is null!",
+ ((LocalResourcesTrackerImpl)pubTracker).getDirsHandler());
+ } finally {
+ dispatcher.stop();
+ delService.stop();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org