You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/09/24 19:07:31 UTC
[09/50] [abbrv] hadoop git commit: YARN-8784.
DockerLinuxContainerRuntime prevents access to distributed cache entries on a
full disk. Contributed by Eric Badger
YARN-8784. DockerLinuxContainerRuntime prevents access to distributed cache entries on a full disk. Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b5838ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b5838ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b5838ed
Branch: refs/heads/YARN-1011
Commit: 6b5838ed3220f992092c7348f92f1d9d0d4a3061
Parents: f6bb1ca
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Sep 19 16:44:51 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Sep 19 16:44:51 2018 -0500
----------------------------------------------------------------------
.../launcher/ContainerLaunch.java | 5 +-
.../launcher/TestContainerLaunch.java | 81 ++++++++++++++++++++
2 files changed, 84 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b5838ed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 2aca5f8..6347d4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -250,12 +250,13 @@ public class ContainerLaunch implements Callable<Integer> {
// accessible by users
pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath);
List<String> localDirs = dirsHandler.getLocalDirs();
+ List<String> localDirsForRead = dirsHandler.getLocalDirsForRead();
List<String> logDirs = dirsHandler.getLogDirs();
- List<String> filecacheDirs = getNMFilecacheDirs(localDirs);
+ List<String> filecacheDirs = getNMFilecacheDirs(localDirsForRead);
List<String> userLocalDirs = getUserLocalDirs(localDirs);
List<String> containerLocalDirs = getContainerLocalDirs(localDirs);
List<String> containerLogDirs = getContainerLogDirs(logDirs);
- List<String> userFilecacheDirs = getUserFilecacheDirs(localDirs);
+ List<String> userFilecacheDirs = getUserFilecacheDirs(localDirsForRead);
List<String> applicationLocalDirs = getApplicationLocalDirs(localDirs,
appIdStr);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b5838ed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index ddf46a6..5714a1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -96,6 +96,7 @@ import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
@@ -107,6 +108,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.Conta
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
@@ -121,6 +123,7 @@ import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.ArgumentCaptor;
public class TestContainerLaunch extends BaseContainerManagerTest {
@@ -2362,4 +2365,82 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
}
}
}
+
+ @Test
+ public void testDistributedCacheDirs() throws Exception {
+ Container container = mock(Container.class);
+ ApplicationId appId =
+ ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ ContainerId containerId = ContainerId
+ .newContainerId(ApplicationAttemptId.newInstance(appId, 1), 1);
+ when(container.getContainerId()).thenReturn(containerId);
+ when(container.getUser()).thenReturn("test");
+
+ when(container.getLocalizedResources())
+ .thenReturn(Collections.<Path, List<String>> emptyMap());
+ Dispatcher dispatcher = mock(Dispatcher.class);
+
+ ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
+ when(clc.getCommands()).thenReturn(Collections.<String>emptyList());
+ when(container.getLaunchContext()).thenReturn(clc);
+
+ @SuppressWarnings("rawtypes")
+ ContainerExitHandler eventHandler =
+ mock(ContainerExitHandler.class);
+ when(dispatcher.getEventHandler()).thenReturn(eventHandler);
+
+ Application app = mock(Application.class);
+ when(app.getAppId()).thenReturn(appId);
+ when(app.getUser()).thenReturn("test");
+
+ Credentials creds = mock(Credentials.class);
+ when(container.getCredentials()).thenReturn(creds);
+
+ ((NMContext) context).setNodeId(NodeId.newInstance("127.0.0.1", HTTP_PORT));
+ ContainerExecutor mockExecutor = mock(ContainerExecutor.class);
+
+ LocalDirsHandlerService mockDirsHandler =
+ mock(LocalDirsHandlerService.class);
+
+ List <String> localDirsForRead = new ArrayList<String>();
+ String localDir1 =
+ new File("target", this.getClass().getSimpleName() + "-localDir1")
+ .getAbsoluteFile().toString();
+ String localDir2 =
+ new File("target", this.getClass().getSimpleName() + "-localDir2")
+ .getAbsoluteFile().toString();
+ localDirsForRead.add(localDir1);
+ localDirsForRead.add(localDir2);
+
+ List <String> localDirs = new ArrayList();
+ localDirs.add(localDir1);
+ Path logPathForWrite = new Path(localDirs.get(0));
+
+ when(mockDirsHandler.areDisksHealthy()).thenReturn(true);
+ when(mockDirsHandler.getLocalDirsForRead()).thenReturn(localDirsForRead);
+ when(mockDirsHandler.getLocalDirs()).thenReturn(localDirs);
+ when(mockDirsHandler.getLogDirs()).thenReturn(localDirs);
+ when(mockDirsHandler.getLogPathForWrite(anyString(),
+ anyBoolean())).thenReturn(logPathForWrite);
+ when(mockDirsHandler.getLocalPathForWrite(anyString()))
+ .thenReturn(logPathForWrite);
+ when(mockDirsHandler.getLocalPathForWrite(anyString(), anyLong(),
+ anyBoolean())).thenReturn(logPathForWrite);
+
+ ContainerLaunch launch = new ContainerLaunch(context, conf, dispatcher,
+ mockExecutor, app, container, mockDirsHandler, containerManager);
+ launch.call();
+
+ ArgumentCaptor <ContainerStartContext> ctxCaptor =
+ ArgumentCaptor.forClass(ContainerStartContext.class);
+ verify(mockExecutor, times(1)).launchContainer(ctxCaptor.capture());
+ ContainerStartContext ctx = ctxCaptor.getValue();
+
+ Assert.assertEquals(StringUtils.join(",",
+ launch.getNMFilecacheDirs(localDirsForRead)),
+ StringUtils.join(",", ctx.getFilecacheDirs()));
+ Assert.assertEquals(StringUtils.join(",",
+ launch.getUserFilecacheDirs(localDirsForRead)),
+ StringUtils.join(",", ctx.getUserFilecacheDirs()));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org